24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
48 Register src, Register dst, Register count, RegSet saved_regs) {
49 if (is_oop) {
50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
51 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
52
53 Label done;
54
55 // Avoid calling runtime if count == 0
56 __ cbz(count, done);
57
58 // Is GC active?
59 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
60 __ ldrb(rscratch1, gc_state);
61 if (ShenandoahSATBBarrier && dest_uninitialized) {
62 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
63 } else {
64 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
65 __ tst(rscratch1, rscratch2);
66 __ br(Assembler::EQ, done);
589 if (is_cae) {
590 // We're falling through to done to indicate success. Success
591 // with is_cae is denoted by returning the value of expected as
592 // result.
593 __ mov(tmp2, expected);
594 }
595
596 __ bind(done);
597 // At entry to done, the Z (EQ) flag is on iff if the CAS
598 // operation was successful. Additionally, if is_cae, tmp2 holds
599 // the value most recently fetched from addr. In this case, success
600 // is denoted by tmp2 matching expected.
601
602 if (is_cae) {
603 __ mov(result, tmp2);
604 } else {
605 __ cset(result, Assembler::EQ);
606 }
607 }
608
609 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
610 Register start, Register count, Register scratch) {
611 assert(ShenandoahCardBarrier, "Should have been checked by caller");
612
613 Label L_loop, L_done;
614 const Register end = count;
615
616 // Zero count? Nothing to do.
617 __ cbz(count, L_done);
618
619 // end = start + count << LogBytesPerHeapOop
620 // last element address to make inclusive
621 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
622 __ sub(end, end, BytesPerHeapOop);
623 __ lsr(start, start, CardTable::card_shift());
624 __ lsr(end, end, CardTable::card_shift());
625
626 // number of bytes to copy
627 __ sub(count, end, start);
628
|
24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #endif
47
48 #define __ masm->
49
50 #ifdef PRODUCT
51 #define BLOCK_COMMENT(str) /* nothing */
52 #else
53 #define BLOCK_COMMENT(str) __ block_comment(str)
54 #endif
55
56 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
57 Register src, Register dst, Register count, RegSet saved_regs) {
58 if (is_oop) {
59 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
60 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
61
62 Label done;
63
64 // Avoid calling runtime if count == 0
65 __ cbz(count, done);
66
67 // Is GC active?
68 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
69 __ ldrb(rscratch1, gc_state);
70 if (ShenandoahSATBBarrier && dest_uninitialized) {
71 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
72 } else {
73 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
74 __ tst(rscratch1, rscratch2);
75 __ br(Assembler::EQ, done);
598 if (is_cae) {
599 // We're falling through to done to indicate success. Success
600 // with is_cae is denoted by returning the value of expected as
601 // result.
602 __ mov(tmp2, expected);
603 }
604
605 __ bind(done);
606 // At entry to done, the Z (EQ) flag is on iff if the CAS
607 // operation was successful. Additionally, if is_cae, tmp2 holds
608 // the value most recently fetched from addr. In this case, success
609 // is denoted by tmp2 matching expected.
610
611 if (is_cae) {
612 __ mov(result, tmp2);
613 } else {
614 __ cset(result, Assembler::EQ);
615 }
616 }
617
618 #ifdef COMPILER2
619 void ShenandoahBarrierSetAssembler::load_ref_barrier_c2(const MachNode* node, MacroAssembler* masm, Register obj, Register addr, bool narrow, bool maybe_null, Register gc_state) {
620 assert_different_registers(obj, addr);
621 BLOCK_COMMENT("load_ref_barrier_c2 {");
622 if (!ShenandoahLoadRefBarrierStubC2::needs_barrier(node)) {
623 return;
624 }
625 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
626 ShenandoahLoadRefBarrierStubC2* const stub = ShenandoahLoadRefBarrierStubC2::create(node, obj, addr, gc_state, noreg, noreg, narrow);
627
628 // Don't preserve the obj across the runtime call, we override it from the
629 // return value anyway.
630 stub->dont_preserve(obj);
631 stub->dont_preserve(gc_state);
632
633 // Check if GC marking is in progress or we are handling a weak reference,
634 // otherwise we don't have to do anything. The code below was optimized to
635 // use less registers and instructions as possible at the expense of always
636 // having a branch instruction. The reason why we use this particular branch
637 // scheme is because the stub entry may be too far for the tbnz to jump to.
638 bool is_strong = (node->barrier_data() & ShenandoahBarrierStrong) != 0;
639 if (is_strong) {
640 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation());
641 __ b(*stub->entry());
642 } else {
643 static_assert(ShenandoahHeap::HAS_FORWARDED_BITPOS == 0, "Relied on in LRB check below.");
644 __ orr(gc_state, gc_state, gc_state, Assembler::LSR, ShenandoahHeap::WEAK_ROOTS_BITPOS);
645 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation());
646 __ b(*stub->entry());
647 }
648
649 __ bind(*stub->continuation());
650 BLOCK_COMMENT("} load_ref_barrier_c2");
651 }
652
653 void ShenandoahBarrierSetAssembler::satb_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register pre_val,
654 Register gc_state, bool encoded_preval) {
655 BLOCK_COMMENT("satb_barrier_c2 {");
656 assert_different_registers(addr, pre_val, rscratch1, rscratch2);
657 if (!ShenandoahSATBBarrierStubC2::needs_barrier(node)) {
658 return;
659 }
660 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
661 ShenandoahSATBBarrierStubC2* const stub = ShenandoahSATBBarrierStubC2::create(node, addr, pre_val, gc_state, encoded_preval);
662
663 // Check if GC marking is in progress, otherwise we don't have to do
664 // anything.
665 __ tstw(gc_state, ShenandoahHeap::MARKING);
666 __ br(Assembler::NE, *stub->entry());
667 __ bind(*stub->continuation());
668 BLOCK_COMMENT("} satb_barrier_c2");
669 }
670
671 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register tmp) {
672 if (!ShenandoahCardBarrier ||
673 (node->barrier_data() & (ShenandoahBarrierCardMark | ShenandoahBarrierCardMarkNotNull)) == 0) {
674 return;
675 }
676
677 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
678 __ lsr(tmp, addr, CardTable::card_shift());
679
680 assert(CardTable::dirty_card_val() == 0, "must be");
681
682 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
683 __ ldr(rscratch1, curr_ct_holder_addr);
684
685 if (UseCondCardMark) {
686 Label L_already_dirty;
687 __ ldrb(rscratch2, Address(tmp, rscratch1));
688 __ cbz(rscratch2, L_already_dirty);
689 __ strb(zr, Address(tmp, rscratch1));
690 __ bind(L_already_dirty);
691 } else {
692 __ strb(zr, Address(tmp, rscratch1));
693 }
694 }
695
696 void ShenandoahBarrierSetAssembler::cmpxchg_oop_c2(const MachNode* node,
697 MacroAssembler* masm,
698 Register addr, Register oldval,
699 Register newval, Register res,
700 Register gc_state, Register tmp,
701 bool acquire, bool release, bool weak, bool exchange) {
702 BLOCK_COMMENT("cmpxchg_oop_c2 {");
703 assert(res != noreg, "need result register");
704 assert_different_registers(oldval, addr, res, gc_state, tmp);
705 assert_different_registers(newval, addr, res, gc_state, tmp);
706
707 // Fast-path: Try to CAS optimistically. If successful, then we are done.
708 // EQ flag set iff success. 'tmp' holds value fetched.
709 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
710 __ cmpxchg(addr, oldval, newval, size, acquire, release, weak, tmp);
711
712 // If we need a boolean result out of CAS, set the flag appropriately. This
713 // would be the final result if we do not go slow.
714 if (!exchange) {
715 __ cset(res, Assembler::EQ);
716 } else {
717 __ mov(res, tmp);
718 }
719
720 if (ShenandoahCASBarrier) {
721 ShenandoahCASBarrierSlowStubC2* const slow_stub =
722 ShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, gc_state, tmp, exchange, acquire, release, weak);
723
724 slow_stub->preserve(gc_state); // this really need to be preserved as we
725 // try to use it in subsequent barriers
726
727 slow_stub->dont_preserve(res); // set at the end, no need to save
728 slow_stub->dont_preserve(oldval); // saved explicitly
729 slow_stub->dont_preserve(tmp); // temp, no need to save
730
731 // On success, we do not need any additional handling.
732 __ br(Assembler::EQ, *slow_stub->continuation());
733
734 // If GC is in progress, it is likely we need additional handling for false negatives.
735 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *slow_stub->continuation());
736 __ b(*slow_stub->entry());
737
738 // Slow stub re-enters with result set correctly.
739 __ bind(*slow_stub->continuation());
740 }
741
742 BLOCK_COMMENT("} cmpxchg_oop_c2");
743 }
744
745 #undef __
746 #define __ masm.
747
748 void ShenandoahLoadRefBarrierStubC2::emit_code(MacroAssembler& masm) {
749 BLOCK_COMMENT("ShenandoahLoadRefBarrierStubC2::emit_code {");
750 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
751 __ bind(*entry());
752 Register obj = _obj;
753 if (_narrow) {
754 __ decode_heap_oop(_tmp1, _obj);
755 obj = _tmp1;
756 }
757 // Weak/phantom loads always need to go to runtime.
758 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
759 // Check for object in cset.
760 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
761 __ lsr(rscratch1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
762 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
763 __ cbz(rscratch2, *continuation());
764 }
765 {
766 SaveLiveRegisters save_registers(&masm, this);
767 if (c_rarg0 != obj) {
768 if (c_rarg0 == _addr) {
769 __ mov(rscratch1, _addr);
770 _addr = rscratch1;
771 }
772 __ mov(c_rarg0, obj);
773 }
774 __ mov(c_rarg1, _addr);
775
776 if (_narrow) {
777 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
778 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
779 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
780 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
781 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
782 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow));
783 }
784 } else {
785 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
786 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
787 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
788 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
789 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
790 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
791 }
792 }
793 __ blr(rscratch1);
794 __ mov(_obj, r0);
795 }
796 if (_narrow) {
797 __ encode_heap_oop(_obj);
798 }
799 __ b(*continuation());
800 BLOCK_COMMENT("} ShenandoahLoadRefBarrierStubC2::emit_code");
801 }
802
803 void ShenandoahSATBBarrierStubC2::emit_code(MacroAssembler& masm) {
804 BLOCK_COMMENT("ShenandoahSATBBarrierStubC2::emit_code {");
805 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
806 __ bind(*entry());
807
808 // The tmp register that we receive is usually a register holding the
809 // "gc_state" which may be required by subsequent memory operations in their
810 // fastpath.
811 RegSet saved = RegSet::of(_tmp);
812 __ push(saved, sp);
813
814 // Do we need to load the previous value?
815 if (_addr != noreg) {
816 __ load_heap_oop(_tmp, Address(_addr, 0), noreg, noreg, AS_RAW);
817 } else {
818 if (_encoded_preval) {
819 __ decode_heap_oop(_tmp, _preval);
820 } else {
821 _tmp = _preval;
822 }
823 }
824
825 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
826 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
827 Label runtime;
828 __ ldr(rscratch1, index);
829 // If buffer is full, call into runtime.
830 __ cbz(rscratch1, runtime);
831
832 // The buffer is not full, store value into it.
833 __ sub(rscratch1, rscratch1, wordSize);
834 __ str(rscratch1, index);
835 __ ldr(rscratch2, buffer);
836 __ str(_tmp, Address(rscratch2, rscratch1));
837 __ pop(saved, sp);
838 __ b(*continuation());
839
840 // Runtime call
841 __ bind(runtime);
842 {
843 SaveLiveRegisters save_registers(&masm, this);
844 __ mov(c_rarg0, _tmp);
845 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2));
846 __ blr(rscratch1);
847 }
848 __ pop(saved, sp);
849 __ b(*continuation());
850 BLOCK_COMMENT("} ShenandoahSATBBarrierStubC2::emit_code");
851 }
852
853 void ShenandoahCASBarrierMidStubC2::emit_code(MacroAssembler& masm) {
854 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
855 __ bind(*entry());
856
857 // Check if CAS result is null. If it is, then we must have a legitimate failure.
858 // This makes loading the fwdptr in the slow-path simpler.
859 __ tst(_result, _result);
860 // In case of !CAE, this has the correct value for legitimate failure (0/false)
861 // in result register.
862 __ br(Assembler::EQ, *continuation());
863
864 // Check if GC is in progress, otherwise we must have a legitimate failure.
865 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
866 __ ldrb(_tmp, gc_state);
867 __ tstw(_tmp, ShenandoahHeap::HAS_FORWARDED);
868 __ br(Assembler::NE, *_slow_stub->entry());
869
870 if (!_cae) {
871 __ mov(_result, 0); // result = false
872 }
873 __ b(*continuation());
874 }
875
876 void ShenandoahCASBarrierSlowStubC2::emit_code(MacroAssembler& masm) {
877 __ bind(*entry());
878
879 // CAS has failed because the value held at addr does not match expected.
880 // This may be a false negative because the version in memory might be
881 // the from-space version of the same object we currently hold to-space
882 // reference for.
883 //
884 // To resolve this, we need to pass the location through the LRB fixup,
885 // this will make sure that the location has only to-space pointers.
886 // To avoid calling into runtime often, we cset-check the object first.
887 // We can inline most of the work here, but there is little point,
888 // as CAS failures over cset locations must be rare. This fast-slow split
889 // matches what we do for normal LRB.
890
891 // Non-strong references should always go to runtime. We do not expect
892 // CASes over non-strong locations.
893 assert((_node->barrier_data() & ShenandoahBarrierStrong) != 0, "Only strong references for CASes");
894
895 Label L_final;
896
897 // (Compressed) failure witness is in _tmp2.
898 // Unpack it and check if it is in collection set.
899 // We need to backup the compressed version to use in the LRB.
900 __ mov(_result, _tmp2);
901 if (UseCompressedOops) {
902 __ decode_heap_oop(_tmp2);
903 }
904
905 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
906 __ lsr(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
907 __ ldrb(_tmp1, Address(_tmp1, _tmp2));
908 __ cbz(_tmp1, L_final);
909
910 {
911 SaveLiveRegisters save_registers(&masm, this);
912 // Load up failure witness again.
913 __ mov(c_rarg0, _result);
914 if (UseCompressedOops) {
915 __ decode_heap_oop(c_rarg0);
916 }
917 __ mov(c_rarg1, _addr_reg);
918
919 if (UseCompressedOops) {
920 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), 2);
921 } else {
922 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), 2);
923 }
924 // We have called LRB to fix up the heap location. We do not care about its
925 // result, as we will just try to CAS the location again.
926 }
927
928 __ bind(L_final);
929
930 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
931 __ cmpxchg(_addr_reg, _expected, _new_val, size, _acquire, _release, _weak, _result);
932
933 if (!_cae) {
934 __ cset(_result, Assembler::EQ);
935 }
936 __ b(*continuation());
937 }
938 #undef __
939 #define __ masm->
940 #endif // COMPILER2
941
942 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
943 Register start, Register count, Register scratch) {
944 assert(ShenandoahCardBarrier, "Should have been checked by caller");
945
946 Label L_loop, L_done;
947 const Register end = count;
948
949 // Zero count? Nothing to do.
950 __ cbz(count, L_done);
951
952 // end = start + count << LogBytesPerHeapOop
953 // last element address to make inclusive
954 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
955 __ sub(end, end, BytesPerHeapOop);
956 __ lsr(start, start, CardTable::card_shift());
957 __ lsr(end, end, CardTable::card_shift());
958
959 // number of bytes to copy
960 __ sub(count, end, start);
961
|