24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44
45 #define __ masm->
46
47 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
48 Register src, Register dst, Register count, RegSet saved_regs) {
49 if (is_oop) {
50 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
51 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
52
53 Label done;
54
55 // Avoid calling runtime if count == 0
56 __ cbz(count, done);
57
58 // Is GC active?
59 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
60 __ ldrb(rscratch1, gc_state);
61 if (ShenandoahSATBBarrier && dest_uninitialized) {
62 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
63 } else {
64 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
65 __ tst(rscratch1, rscratch2);
66 __ br(Assembler::EQ, done);
585 if (is_cae) {
586 // We're falling through to done to indicate success. Success
587 // with is_cae is denoted by returning the value of expected as
588 // result.
589 __ mov(tmp2, expected);
590 }
591
592 __ bind(done);
593 // At entry to done, the Z (EQ) flag is on iff if the CAS
594 // operation was successful. Additionally, if is_cae, tmp2 holds
595 // the value most recently fetched from addr. In this case, success
596 // is denoted by tmp2 matching expected.
597
598 if (is_cae) {
599 __ mov(result, tmp2);
600 } else {
601 __ cset(result, Assembler::EQ);
602 }
603 }
604
605 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
606 Register start, Register count, Register scratch) {
607 assert(ShenandoahCardBarrier, "Should have been checked by caller");
608
609 Label L_loop, L_done;
610 const Register end = count;
611
612 // Zero count? Nothing to do.
613 __ cbz(count, L_done);
614
615 // end = start + count << LogBytesPerHeapOop
616 // last element address to make inclusive
617 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
618 __ sub(end, end, BytesPerHeapOop);
619 __ lsr(start, start, CardTable::card_shift());
620 __ lsr(end, end, CardTable::card_shift());
621
622 // number of bytes to copy
623 __ sub(count, end, start);
624
|
24 */
25
26 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
27 #include "gc/shenandoah/mode/shenandoahMode.hpp"
28 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
30 #include "gc/shenandoah/shenandoahForwarding.hpp"
31 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
32 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
33 #include "gc/shenandoah/shenandoahRuntime.hpp"
34 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
35 #include "interpreter/interp_masm.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #ifdef COMPILER1
40 #include "c1/c1_LIRAssembler.hpp"
41 #include "c1/c1_MacroAssembler.hpp"
42 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
43 #endif
44 #ifdef COMPILER2
45 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
46 #endif
47
48 #define __ masm->
49
50 #ifdef PRODUCT
51 #define BLOCK_COMMENT(str) /* nothing */
52 #else
53 #define BLOCK_COMMENT(str) __ block_comment(str)
54 #endif
55
56 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
57 Register src, Register dst, Register count, RegSet saved_regs) {
58 if (is_oop) {
59 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
60 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
61
62 Label done;
63
64 // Avoid calling runtime if count == 0
65 __ cbz(count, done);
66
67 // Is GC active?
68 Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
69 __ ldrb(rscratch1, gc_state);
70 if (ShenandoahSATBBarrier && dest_uninitialized) {
71 __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
72 } else {
73 __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);
74 __ tst(rscratch1, rscratch2);
75 __ br(Assembler::EQ, done);
594 if (is_cae) {
595 // We're falling through to done to indicate success. Success
596 // with is_cae is denoted by returning the value of expected as
597 // result.
598 __ mov(tmp2, expected);
599 }
600
601 __ bind(done);
602 // At entry to done, the Z (EQ) flag is on iff if the CAS
603 // operation was successful. Additionally, if is_cae, tmp2 holds
604 // the value most recently fetched from addr. In this case, success
605 // is denoted by tmp2 matching expected.
606
607 if (is_cae) {
608 __ mov(result, tmp2);
609 } else {
610 __ cset(result, Assembler::EQ);
611 }
612 }
613
614 #ifdef COMPILER2
615 void ShenandoahBarrierSetAssembler::load_ref_barrier_c2(const MachNode* node, MacroAssembler* masm, Register obj, Register addr, bool narrow, bool maybe_null, Register gc_state) {
616 assert_different_registers(obj, addr);
617 BLOCK_COMMENT("load_ref_barrier_c2 {");
618 if (!ShenandoahLoadRefBarrierStubC2::needs_barrier(node)) {
619 return;
620 }
621 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
622 ShenandoahLoadRefBarrierStubC2* const stub = ShenandoahLoadRefBarrierStubC2::create(node, obj, addr, gc_state, noreg, noreg, narrow);
623
624 // Don't preserve the obj across the runtime call, we override it from the
625 // return value anyway.
626 stub->dont_preserve(obj);
627 stub->dont_preserve(gc_state);
628
629 // Check if GC marking is in progress or we are handling a weak reference,
630 // otherwise we don't have to do anything. The code below was optimized to
631 // use less registers and instructions as possible at the expense of always
632 // having a branch instruction. The reason why we use this particular branch
633 // scheme is because the stub entry may be too far for the tbnz to jump to.
634 bool is_strong = (node->barrier_data() & ShenandoahBarrierStrong) != 0;
635 if (is_strong) {
636 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation());
637 __ b(*stub->entry());
638 } else {
639 static_assert(ShenandoahHeap::HAS_FORWARDED_BITPOS == 0, "Relied on in LRB check below.");
640 __ orr(gc_state, gc_state, gc_state, Assembler::LSR, ShenandoahHeap::WEAK_ROOTS_BITPOS);
641 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *stub->continuation());
642 __ b(*stub->entry());
643 }
644
645 __ bind(*stub->continuation());
646 BLOCK_COMMENT("} load_ref_barrier_c2");
647 }
648
649 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
650 Register dst, bool dst_narrow,
651 Register src, bool src_narrow,
652 Register tmp, Register pre_val,
653 bool is_volatile) {
654
655 Address gcs_addr(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
656 __ ldrb(tmp, gcs_addr);
657
658 satb_barrier_c2(node, masm, dst, pre_val, tmp, src_narrow);
659
660 card_barrier_c2(node, masm, dst, tmp);
661
662 // Need to encode into tmp, because we cannot clobber src.
663 // TODO: Maybe there is a matcher way to test that src is unused after this?
664 if (dst_narrow && !src_narrow) {
665 __ mov(tmp, src);
666 if (ShenandoahStoreBarrierStubC2::src_not_null(node)) {
667 __ encode_heap_oop_not_null(tmp);
668 } else {
669 __ encode_heap_oop(tmp);
670 }
671 src = tmp;
672 }
673
674 // Do the actual store
675 if (dst_narrow) {
676 if (is_volatile) {
677 __ stlrw(src, dst);
678 } else {
679 __ strw(src, dst);
680 }
681 } else {
682 if (is_volatile) {
683 __ stlr(src, dst);
684 } else {
685 __ str(src, dst);
686 }
687 }
688 }
689
690 void ShenandoahBarrierSetAssembler::satb_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register pre_val,
691 Register gc_state, bool encoded_preval) {
692 BLOCK_COMMENT("satb_barrier_c2 {");
693 assert_different_registers(addr, pre_val, rscratch1, rscratch2);
694 if (!ShenandoahSATBBarrierStubC2::needs_barrier(node)) {
695 return;
696 }
697 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
698 ShenandoahSATBBarrierStubC2* const stub = ShenandoahSATBBarrierStubC2::create(node, addr, pre_val, gc_state, encoded_preval);
699
700 // Check if GC marking is in progress, otherwise we don't have to do
701 // anything.
702 __ tstw(gc_state, ShenandoahHeap::MARKING);
703 __ br(Assembler::NE, *stub->entry());
704 __ bind(*stub->continuation());
705 BLOCK_COMMENT("} satb_barrier_c2");
706 }
707
708 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Register addr, Register tmp) {
709 if (!ShenandoahCardBarrier ||
710 (node->barrier_data() & ShenandoahBarrierCardMark) == 0) {
711 return;
712 }
713
714 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
715 __ lsr(tmp, addr, CardTable::card_shift());
716
717 assert(CardTable::dirty_card_val() == 0, "must be");
718
719 Address curr_ct_holder_addr(rthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
720 __ ldr(rscratch1, curr_ct_holder_addr);
721
722 if (UseCondCardMark) {
723 Label L_already_dirty;
724 __ ldrb(rscratch2, Address(tmp, rscratch1));
725 __ cbz(rscratch2, L_already_dirty);
726 __ strb(zr, Address(tmp, rscratch1));
727 __ bind(L_already_dirty);
728 } else {
729 __ strb(zr, Address(tmp, rscratch1));
730 }
731 }
732
733 void ShenandoahBarrierSetAssembler::cmpxchg_oop_c2(const MachNode* node,
734 MacroAssembler* masm,
735 Register addr, Register oldval,
736 Register newval, Register res,
737 Register gc_state, Register tmp,
738 bool acquire, bool release, bool weak, bool exchange) {
739 assert(res != noreg, "need result register");
740 assert_different_registers(oldval, addr, res, gc_state, tmp);
741 assert_different_registers(newval, addr, res, gc_state, tmp);
742
743 // Fast-path: Try to CAS optimistically. If successful, then we are done.
744 // EQ flag set iff success. 'tmp' holds value fetched.
745 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
746 __ cmpxchg(addr, oldval, newval, size, acquire, release, weak, tmp);
747
748 // If we need a boolean result out of CAS, set the flag appropriately. This
749 // would be the final result if we do not go slow.
750 if (!exchange) {
751 __ cset(res, Assembler::EQ);
752 } else {
753 __ mov(res, tmp);
754 }
755
756 if (ShenandoahCASBarrier) {
757 ShenandoahCASBarrierSlowStubC2* const slow_stub =
758 ShenandoahCASBarrierSlowStubC2::create(node, addr, oldval, newval, res, gc_state, tmp, exchange, acquire, release, weak);
759
760 slow_stub->preserve(gc_state); // this really need to be preserved as we
761 // try to use it in subsequent barriers
762
763 slow_stub->dont_preserve(res); // set at the end, no need to save
764 slow_stub->dont_preserve(oldval); // saved explicitly
765 slow_stub->dont_preserve(tmp); // temp, no need to save
766
767 // On success, we do not need any additional handling.
768 __ br(Assembler::EQ, *slow_stub->continuation());
769
770 // If GC is in progress, it is likely we need additional handling for false negatives.
771 __ tbz(gc_state, ShenandoahHeap::HAS_FORWARDED_BITPOS, *slow_stub->continuation());
772 __ b(*slow_stub->entry());
773
774 // Slow stub re-enters with result set correctly.
775 __ bind(*slow_stub->continuation());
776 }
777 }
778
779 #undef __
780 #define __ masm.
781
782 void ShenandoahLoadRefBarrierStubC2::emit_code(MacroAssembler& masm) {
783 BLOCK_COMMENT("ShenandoahLoadRefBarrierStubC2::emit_code {");
784 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
785 __ bind(*entry());
786 Register obj = _obj;
787 if (_narrow) {
788 __ decode_heap_oop(_tmp1, _obj);
789 obj = _tmp1;
790 }
791 // Weak/phantom loads always need to go to runtime.
792 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
793 // Check for object in cset.
794 __ mov(rscratch2, ShenandoahHeap::in_cset_fast_test_addr());
795 __ lsr(rscratch1, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
796 __ ldrb(rscratch2, Address(rscratch2, rscratch1));
797 __ cbz(rscratch2, *continuation());
798 }
799 {
800 SaveLiveRegisters save_registers(&masm, this);
801 if (c_rarg0 != obj) {
802 if (c_rarg0 == _addr) {
803 __ mov(rscratch1, _addr);
804 _addr = rscratch1;
805 }
806 __ mov(c_rarg0, obj);
807 }
808 __ mov(c_rarg1, _addr);
809
810 if (_narrow) {
811 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
812 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow));
813 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
814 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow));
815 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
816 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom_narrow));
817 }
818 } else {
819 if ((_node->barrier_data() & ShenandoahBarrierStrong) != 0) {
820 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong));
821 } else if ((_node->barrier_data() & ShenandoahBarrierWeak) != 0) {
822 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak));
823 } else if ((_node->barrier_data() & ShenandoahBarrierPhantom) != 0) {
824 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom));
825 }
826 }
827 __ blr(rscratch1);
828 __ mov(_obj, r0);
829 }
830 if (_narrow) {
831 __ encode_heap_oop(_obj);
832 }
833 __ b(*continuation());
834 BLOCK_COMMENT("} ShenandoahLoadRefBarrierStubC2::emit_code");
835 }
836
837 void ShenandoahSATBBarrierStubC2::emit_code(MacroAssembler& masm) {
838 BLOCK_COMMENT("ShenandoahSATBBarrierStubC2::emit_code {");
839 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
840 __ bind(*entry());
841
842 // The tmp register that we receive is usually a register holding the
843 // "gc_state" which may be required by subsequent memory operations in their
844 // fastpath.
845 RegSet saved = RegSet::of(_tmp);
846 __ push(saved, sp);
847
848 // Do we need to load the previous value?
849 if (_addr != noreg) {
850 __ load_heap_oop(_tmp, Address(_addr, 0), noreg, noreg, AS_RAW);
851 } else {
852 if (_encoded_preval) {
853 __ decode_heap_oop(_tmp, _preval);
854 } else {
855 _tmp = _preval;
856 }
857 }
858
859 Address index(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
860 Address buffer(rthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
861 Label runtime;
862 __ ldr(rscratch1, index);
863 // If buffer is full, call into runtime.
864 __ cbz(rscratch1, runtime);
865
866 // The buffer is not full, store value into it.
867 __ sub(rscratch1, rscratch1, wordSize);
868 __ str(rscratch1, index);
869 __ ldr(rscratch2, buffer);
870 __ str(_tmp, Address(rscratch2, rscratch1));
871 __ pop(saved, sp);
872 __ b(*continuation());
873
874 // Runtime call
875 __ bind(runtime);
876 {
877 SaveLiveRegisters save_registers(&masm, this);
878 __ mov(c_rarg0, _tmp);
879 __ mov(rscratch1, CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre_c2));
880 __ blr(rscratch1);
881 }
882 __ pop(saved, sp);
883 __ b(*continuation());
884 BLOCK_COMMENT("} ShenandoahSATBBarrierStubC2::emit_code");
885 }
886
887 void ShenandoahLoadBarrierStubC2::emit_code(MacroAssembler& masm) {
888 Unimplemented();
889 }
890
891 void ShenandoahStoreBarrierStubC2::emit_code(MacroAssembler& masm) {
892 Unimplemented();
893 }
894
895 void ShenandoahCASBarrierSlowStubC2::emit_code(MacroAssembler& masm) {
896 __ bind(*entry());
897
898 // CAS has failed because the value held at addr does not match expected.
899 // This may be a false negative because the version in memory might be
900 // the from-space version of the same object we currently hold to-space
901 // reference for.
902 //
903 // To resolve this, we need to pass the location through the LRB fixup,
904 // this will make sure that the location has only to-space pointers.
905 // To avoid calling into runtime often, we cset-check the object first.
906 // We can inline most of the work here, but there is little point,
907 // as CAS failures over cset locations must be rare. This fast-slow split
908 // matches what we do for normal LRB.
909
910 // Non-strong references should always go to runtime. We do not expect
911 // CASes over non-strong locations.
912 assert((_node->barrier_data() & ShenandoahBarrierStrong) != 0, "Only strong references for CASes");
913
914 Label L_final;
915
916 // (Compressed) failure witness is in _tmp2.
917 // Unpack it and check if it is in collection set.
918 // We need to backup the compressed version to use in the LRB.
919 __ mov(_result, _tmp2);
920 if (UseCompressedOops) {
921 __ decode_heap_oop(_tmp2);
922 }
923
924 __ mov(_tmp1, ShenandoahHeap::in_cset_fast_test_addr());
925 __ lsr(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
926 __ ldrb(_tmp1, Address(_tmp1, _tmp2));
927 __ cbz(_tmp1, L_final);
928
929 {
930 SaveLiveRegisters save_registers(&masm, this);
931 // Load up failure witness again.
932 __ mov(c_rarg0, _result);
933 if (UseCompressedOops) {
934 __ decode_heap_oop(c_rarg0);
935 }
936 __ mov(c_rarg1, _addr_reg);
937
938 if (UseCompressedOops) {
939 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong_narrow), 2);
940 } else {
941 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_strong), 2);
942 }
943 // We have called LRB to fix up the heap location. We do not care about its
944 // result, as we will just try to CAS the location again.
945 }
946
947 __ bind(L_final);
948
949 Assembler::operand_size size = UseCompressedOops ? Assembler::word : Assembler::xword;
950 __ cmpxchg(_addr_reg, _expected, _new_val, size, _acquire, _release, _weak, _result);
951
952 if (!_cae) {
953 __ cset(_result, Assembler::EQ);
954 }
955 __ b(*continuation());
956 }
957 #undef __
958 #define __ masm->
959 #endif // COMPILER2
960
961 void ShenandoahBarrierSetAssembler::gen_write_ref_array_post_barrier(MacroAssembler* masm, DecoratorSet decorators,
962 Register start, Register count, Register scratch) {
963 assert(ShenandoahCardBarrier, "Should have been checked by caller");
964
965 Label L_loop, L_done;
966 const Register end = count;
967
968 // Zero count? Nothing to do.
969 __ cbz(count, L_done);
970
971 // end = start + count << LogBytesPerHeapOop
972 // last element address to make inclusive
973 __ lea(end, Address(start, count, Address::lsl(LogBytesPerHeapOop)));
974 __ sub(end, end, BytesPerHeapOop);
975 __ lsr(start, start, CardTable::card_shift());
976 __ lsr(end, end, CardTable::card_shift());
977
978 // number of bytes to copy
979 __ sub(count, end, start);
980
|