24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
48 Register src, Register dst, Register count, RegSet saved_regs) {
49 if (is_oop) {
50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
51 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
52
53 Label done;
54
55 // Avoid calling runtime if count == 0
56 __ cbz(count, done);
57
58 // Is GC active?
59 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
60 __ ldrb(rscratch1, gc_state);
61 if (ShenandoahSATBBarrier && dest_uninitialized) {
62 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
63 } else {
589 if (is_cae) {
590 // We're falling through to done to indicate success. Success
591 // with is_cae is denoted by returning the value of expected as
592 // result.
593 __ mov(tmp2, expected);
594 }
595
596 __ bind(done);
597 // At entry to done, the Z (EQ) flag is on iff if the CAS
598 // operation was successful. Additionally, if is_cae, tmp2 holds
599 // the value most recently fetched from addr. In this case, success
600 // is denoted by tmp2 matching expected.
601
602 if (is_cae) {
603 __ mov(result, tmp2);
604 } else {
605 __ cset(result, Assembler::EQ);
606 }
607 }
608
609 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
610 Register start, Register count, Register scratch) {
611 assert(ShenandoahCardBarrier, "Should have been checked by caller");
612
613 Label L_loop, L_done;
614 const Register end = count;
615
616 // Zero count? Nothing to do.
617 __ cbz(count, L_done);
618
619 // end = start + count << LogBytesPerHeapOop
620 // last element address to make inclusive
621 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
622 __ sub(end, end, BytesPerHeapOop);
623 __ lsr(start, start, CardTable::card_shift());
624 __ lsr(end, end, CardTable::card_shift());
625
626 // number of bytes to copy
627 __ sub(count, end, start);
628
|
24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #endif
47
48 #define __ masm->
49
50 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
51 Register src, Register dst, Register count, RegSet saved_regs) {
52 if (is_oop) {
53 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
54 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
55
56 Label done;
57
58 // Avoid calling runtime if count == 0
59 __ cbz(count, done);
60
61 // Is GC active?
62 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
63 __ ldrb(rscratch1, gc_state);
64 if (ShenandoahSATBBarrier && dest_uninitialized) {
65 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
66 } else {
592 if (is_cae) {
593 // We're falling through to done to indicate success. Success
594 // with is_cae is denoted by returning the value of expected as
595 // result.
596 __ mov(tmp2, expected);
597 }
598
599 __ bind(done);
600 // At entry to done, the Z (EQ) flag is on iff if the CAS
601 // operation was successful. Additionally, if is_cae, tmp2 holds
602 // the value most recently fetched from addr. In this case, success
603 // is denoted by tmp2 matching expected.
604
605 if (is_cae) {
606 __ mov(result, tmp2);
607 } else {
608 __ cset(result, Assembler::EQ);
609 }
610 }
611
612 #ifdef COMPILER2
613 void ShenandoahBarrierSetAssembler::load_ref_barrier_c2(const MachNode* node, MacroAssembler* masm, Register obj, Register addr, Register tmp, bool narrow, bool maybe_null) {
614 if (!ShenandoahLoadRefBarrierStubC2::needs_barrier(node)) {
615 return;
616 }
617 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
618 ShenandoahLoadRefBarrierStubC2* const stub = ShenandoahLoadRefBarrierStubC2::create(node, obj, addr, tmp, noreg, noreg, narrow);
619
620 // Don't preserve the obj across the runtime call, we override it from the return value anyway.
621 stub->dont_preserve(obj);
622
623 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
624 __ ldrb(rscratch1, gc_state);
625
626 // Check if GC marking is in progress or we are handling a weak reference, otherwise we don't have to do anything.
627 bool is_strong = (node->barrier_data() & ShenandoahBarrierStrong) != 0;
628 if (is_strong) {
629 __ tbnz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->entry());
630 } else {
631 static_assert(ShenandoahHeap::HAS_FORWARDED_BITPOS == 0, "Relied on in LRB check below.");
632 __ orr(tmp, rscratch1, rscratch1, Assembler::LSR, ShenandoahHeap::WEAK_ROOTS_BITPOS);
633 __ tbnz(tmp, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->entry());
634 }
635
636 __ bind(*stub->continuation());
637 }
638
639 void ShenandoahBarrierSetAssembler::load_ref_barrier_c3(const MachNode* node, MacroAssembler* masm, Register obj, Register addr, Register tmp, bool narrow, bool maybe_null, Register gc_state) {
640 if (!ShenandoahLoadRefBarrierStubC2::needs_barrier(node)) {
641 return;
642 }
643 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
644 ShenandoahLoadRefBarrierStubC2* const stub = ShenandoahLoadRefBarrierStubC2::create(node, obj, addr, tmp, noreg, noreg, narrow);
645
646 // Don't preserve the obj across the runtime call, we override it from the return value anyway.
647 stub->dont_preserve(obj);
648
649 // Check if GC marking is in progress or we are handling a weak reference, otherwise we don't have to do anything.
650 bool is_strong = (node->barrier_data() & ShenandoahBarrierStrong) != 0;
651 if (is_strong) {
652 __ tbnz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->entry());
653 } else {
654 static_assert(ShenandoahHeap::HAS_FORWARDED_BITPOS == 0, "Relied on in LRB check below.");
655 __ orr(tmp, gc_state, gc_state, Assembler::LSR, ShenandoahHeap::WEAK_ROOTS_BITPOS);
656 __ tbnz(tmp, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->entry());
657 }
658
659 __ bind(*stub->continuation());
660 }
661
662 void ShenandoahBarrierSetAssembler::satb_barrier_c3(const MachNode* node, MacroAssembler* masm, Register addr, Register pre_val, Register gc_state) {
663 assert_different_registers(addr, pre_val);
664 if (!ShenandoahSATBBarrierStubC2::needs_barrier(node)) {
665 return;
666 }
667
668 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
669 ShenandoahSATBBarrierStubC2* const stub = ShenandoahSATBBarrierStubC2::create(node, addr, pre_val);
670
671 // Check if GC marking is in progress, otherwise we don't have to do anything.
672 __ tstw(gc_state, ShenandoahHeap::MARKING);
673 __ br(Assembler::NE, *stub->entry());
674 __ bind(*stub->continuation());
675 }
676
677 void ShenandoahBarrierSetAssembler::satb_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register pre_val) {
678 assert_different_registers(addr, pre_val);
679 if (!ShenandoahSATBBarrierStubC2::needs_barrier(node)) {
680 return;
681 }
682 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
683 ShenandoahSATBBarrierStubC2* const stub = ShenandoahSATBBarrierStubC2::create(node, addr, pre_val);
684
685 // Check if GC marking is in progress, otherwise we don't have to do anything.
686 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
687 __ ldrb(rscratch1, gc_state);
688 __ tstw(rscratch1, ShenandoahHeap::MARKING);
689 __ br(Assembler::NE, *stub->entry());
690 __ bind(*stub->continuation());
691 }
692
693 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register tmp) {
694 if (!ShenandoahCardBarrier ||
695 (node->barrier_data() & (ShenandoahBarrierCardMark | ShenandoahBarrierCardMarkNotNull)) == 0) {
696 return;
697 }
698
699 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
700 __ lsr(tmp, addr, CardTable::card_shift());
701
702 assert(CardTable::dirty_card_val() == 0, "must be");
703
704 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
705 __ ldr(rscratch1, curr_ct_holder_addr);
706
707 if (UseCondCardMark) {
708 Label L_already_dirty;
709 __ ldrb(rscratch2, Address(tmp, rscratch1));
710 __ cbz(rscratch2, L_already_dirty);
711 __ strb(zr, Address(tmp, rscratch1));
712 __ bind(L_already_dirty);
713 } else {
714 __ strb(zr, Address(tmp, rscratch1));
715 }
716 }
717
718 void ShenandoahBarrierSetAssembler::cmpxchg_oop_c2(const MachNode* node,
719 MacroAssembler* masm,
720 Register addr,
721 Register expected,
722 Register new_val,
723 Register result,
724 bool acquire, bool release, bool weak,
725 bool is_cae) {
726 Register tmp = rscratch2;
727 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
728
729 assert_different_registers(addr, expected, result, tmp);
730 assert_different_registers(addr, new_val, result, tmp);
731
732 ShenandoahCASBarrierSlowStubC2* const slow_stub = ShenandoahCASBarrierSlowStubC2::create(node, addr, expected, new_val, result, tmp, is_cae, acquire, release, weak);
733 ShenandoahCASBarrierMidStubC2* const mid_stub = ShenandoahCASBarrierMidStubC2::create(node, slow_stub, expected, result, tmp, is_cae);
734
735 // Step 1. Fast-path.
736 //
737 // Try to CAS with given arguments. If successful, then we are done.
738 __ cmpxchg(addr, expected, new_val, size, acquire, release, weak, result);
739 // EQ flag set iff success. result holds value fetched.
740
741 __ br(Assembler::NE, *mid_stub->entry());
742
743 // Slow-stub re-enters with condition flags according to CAS, we may need to
744 // set result accordingly.
745 __ bind(*slow_stub->continuation());
746 if (!is_cae) {
747 __ cset(result, Assembler::EQ);
748 }
749
750 // Mid-stub re-enters with result set correctly.
751 __ bind(*mid_stub->continuation());
752 }
753
754 #undef __
755 #define __ masm.
756
757 void ShenandoahLoadRefBarrierStubC2::emit_code(MacroAssembler& masm) {
758 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
759 __ bind(*entry());
760 Register obj = _obj;
761 if (_narrow) {
762 __ decode_heap_oop(_tmp1, _obj);
763 obj = _tmp1;
764 }
765 // Weak/phantom loads always need to go to runtime.
766 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
767 // Check for object in cset.
768 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
769 __ lsr(rscratch1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
770 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
771 __ cbz(rscratch2, *continuation());
772 }
773 {
774 SaveLiveRegisters save_registers(&masm, this);
775 if (c_rarg0 != obj) {
776 if (c_rarg0 == _addr) {
777 __ mov(rscratch1, _addr);
778 _addr = rscratch1;
779 }
780 __ mov(c_rarg0, obj);
781 }
782 __ mov(c_rarg1, _addr);
783
784 if (_narrow) {
785 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
786 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
787 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
788 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
789 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
790 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow));
791 }
792 } else {
793 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
794 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
795 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
796 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
797 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
798 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
799 }
800 }
801 __ blr(rscratch1);
802 __ mov(_obj, r0);
803 }
804 if (_narrow) {
805 __ encode_heap_oop(_obj);
806 }
807 __ b(*continuation());
808 }
809
810 void ShenandoahSATBBarrierStubC2::emit_code(MacroAssembler& masm) {
811 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
812 __ bind(*entry());
813 // Do we need to load the previous value?
814 if (_addr != noreg) {
815 __ load_heap_oop(_preval, Address(_addr, 0), noreg, noreg, AS_RAW);
816 }
817 // Is the previous value null?
818 // __ cbz(_preval, *continuation());
819
820 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
821 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
822 Label runtime;
823 __ ldr(rscratch1, index);
824 // If buffer is full, call into runtime.
825 __ cbz(rscratch1, runtime);
826
827 // The buffer is not full, store value into it.
828 __ sub(rscratch1, rscratch1, wordSize);
829 __ str(rscratch1, index);
830 __ ldr(rscratch2, buffer);
831 __ str(_preval, Address(rscratch2, rscratch1));
832 __ b(*continuation());
833
834 // Runtime call
835 __ bind(runtime);
836 {
837 SaveLiveRegisters save_registers(&masm, this);
838 if (c_rarg0 != _preval) {
839 __ mov(c_rarg0, _preval);
840 }
841 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2));
842 __ blr(rscratch1);
843 }
844 __ b(*continuation());
845 }
846
847 void ShenandoahCASBarrierMidStubC2::emit_code(MacroAssembler& masm) {
848 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
849 __ bind(*entry());
850
851 // Check if CAS result is null. If it is, then we must have a legitimate failure.
852 // This makes loading the fwdptr in the slow-path simpler.
853 __ tst(_result, _result);
854 // In case of !CAE, this has the correct value for legitimate failure (0/false)
855 // in result register.
856 __ br(Assembler::EQ, *continuation());
857
858 // Check if GC is in progress, otherwise we must have a legitimate failure.
859 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
860 __ ldrb(_tmp, gc_state);
861 __ tstw(_tmp, ShenandoahHeap::HAS_FORWARDED);
862 __ br(Assembler::NE, *_slow_stub->entry());
863
864 if (!_cae) {
865 __ mov(_result, 0); // result = false
866 }
867 __ b(*continuation());
868 }
869
870 void ShenandoahCASBarrierSlowStubC2::emit_code(MacroAssembler& masm) {
871 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
872 __ bind(*entry());
873 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
874
875 // Step 2. CAS has failed because the value held at addr does not
876 // match expected. This may be a false negative because the value fetched
877 // from addr (now held in result) may be a from-space pointer to the
878 // original copy of same object referenced by to-space pointer expected.
879 //
880 // To resolve this, it suffices to find the forward pointer associated
881 // with fetched value. If this matches expected, retry CAS with new
882 // parameters. If this mismatches, then we have a legitimate
883 // failure, and we're done.
884
885 // overwrite tmp with from-space pointer fetched from memory
886 __ mov(_tmp1, _result);
887
888 if (UseCompressedOops) {
889 // Decode tmp in order to resolve its forward pointer
890 __ decode_heap_oop_not_null(_tmp1, _tmp1);
891 }
892
893 // Load/decode forwarding pointer.
894 __ ldr(_tmp1, Address(_tmp1, oopDesc::mark_offset_in_bytes()));
895 // Negate the mark-word. This allows us to test lowest 2 bits easily while preserving the upper bits.
896 __ eon(_tmp1, _tmp1, zr);
897 __ ands(zr, _tmp1, markWord::lock_mask_in_place);
898 // Not forwarded, must have a legit CAS failure.
899 __ br(Assembler::NE, *continuation());
900 // Set the lowest two bits. This is equivalent to clearing the two bits after
901 // the subsequent inversion.
902 __ orr(_tmp1, _tmp1, markWord::marked_value);
903 // And invert back to get the forwardee.
904 __ eon(_tmp1, _tmp1, zr);
905
906 if (UseCompressedOops) {
907 // Encode tmp to compare against expected.
908 __ encode_heap_oop_not_null(_tmp1, _tmp1);
909 }
910
911 // Does forwarded value of fetched from-space pointer match original
912 // value of expected? If result holds null, this comparison will fail
913 // because we know from step1 that expected is not null. There is
914 // no need for a separate test for result (the value originally held
915 // in memory) equal to null.
916 __ cmp(_tmp1, _expected);
917
918 // If not, then the failure was legitimate and we're done.
919 // Branching to continuation with NE condition denotes failure.
920 __ br(Assembler::NE, *continuation());
921
922 // Fall through to step 3.
923
924 // Step 3. We've confirmed that the value originally held in memory
925 // (now held in result) pointed to from-space version of original
926 // expected value. Try the CAS again with the from-space expected
927 // value. If it now succeeds, we're good.
928 //
929 // Note: result holds encoded from-space pointer that matches to-space
930 // object residing at expected. result is the new "expected".
931
932 // Note that macro implementation of __cmpxchg cannot use same register
933 // tmp2 for result and expected since it overwrites result before it
934 // compares result with expected.
935 __ mov(_tmp1, _result);
936 __ cmpxchg(_addr_reg, _tmp1, _new_val, size, _acquire, _release, _weak, _result);
937 // EQ flag set iff success. result holds value fetched, rscratch1 clobbered.
938
939 // If fetched value did not equal the new expected, this could
940 // still be a false negative because some other thread may have
941 // newly overwritten the memory value with its to-space equivalent.
942 __ br(Assembler::EQ, *continuation());
943
944 // Step 4. Retry CAS with original to-space expected.
945 __ cmpxchg(_addr_reg, _expected, _new_val, size, _acquire, _release, _weak, _result);
946
947 __ b(*continuation());
948 }
949 #undef __
950 #define __ masm->
951 #endif // COMPILER2
952
953 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
954 Register start, Register count, Register scratch) {
955 assert(ShenandoahCardBarrier, "Should have been checked by caller");
956
957 Label L_loop, L_done;
958 const Register end = count;
959
960 // Zero count? Nothing to do.
961 __ cbz(count, L_done);
962
963 // end = start + count << LogBytesPerHeapOop
964 // last element address to make inclusive
965 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
966 __ sub(end, end, BytesPerHeapOop);
967 __ lsr(start, start, CardTable::card_shift());
968 __ lsr(end, end, CardTable::card_shift());
969
970 // number of bytes to copy
971 __ sub(count, end, start);
972
|