25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
49 Register src, Register dst, Register count, RegSet saved_regs) {
50 if (is_oop) {
51 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
52 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
53
54 Label done;
55
56 // Avoid calling runtime if count == 0
57 __ beqz(count, done);
58
59 // Is GC active?
60 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
61 assert_different_registers(src, dst, count, t0);
62
63 __ lbu(t0, gc_state);
64 if (ShenandoahSATBBarrier && dest_uninitialized) {
760 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
761 } else {
762 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
763 }
764 } else {
765 assert(is_phantom, "only remaining strength");
766 assert(is_native, "phantom must only be called off-heap");
767 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
768 }
769 __ rt_call(target);
770 __ mv(t0, x10);
771 __ pop_call_clobbered_registers();
772 __ mv(x10, t0);
773
774 __ epilogue();
775 }
776
777 #undef __
778
779 #endif // COMPILER1
|
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interp_masm.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #include "opto/output.hpp"
48 #include "utilities/population_count.hpp"
49 #include "utilities/powerOfTwo.hpp"
50 #endif
51
52 #define __ masm->
53
54 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, bool is_oop,
55 Register src, Register dst, Register count, RegSet saved_regs) {
56 if (is_oop) {
57 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
58 if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
59
60 Label done;
61
62 // Avoid calling runtime if count == 0
63 __ beqz(count, done);
64
65 // Is GC active?
66 Address gc_state(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
67 assert_different_registers(src, dst, count, t0);
68
69 __ lbu(t0, gc_state);
70 if (ShenandoahSATBBarrier && dest_uninitialized) {
766 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow);
767 } else {
768 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak);
769 }
770 } else {
771 assert(is_phantom, "only remaining strength");
772 assert(is_native, "phantom must only be called off-heap");
773 target = CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom);
774 }
775 __ rt_call(target);
776 __ mv(t0, x10);
777 __ pop_call_clobbered_registers();
778 __ mv(x10, t0);
779
780 __ epilogue();
781 }
782
783 #undef __
784
785 #endif // COMPILER1
786
787 #ifdef COMPILER2
788
789 #define __ masm->
790
791 static bool needs_acquiring_load_reserved(const MachNode* n) {
792 assert(n->is_CAS(true), "expecting a compare and swap");
793 if (n->is_CAS(false)) {
794 assert(n->has_trailing_membar(), "expected trailing membar");
795 } else {
796 return n->has_trailing_membar();
797 }
798 return true;
799 }
800
801 void ShenandoahBarrierStubC2::gc_state_check_c2(MacroAssembler* masm,
802 Register gcstate,
803 const unsigned char test_state,
804 ShenandoahBarrierStubC2* slow_stub) {
805 if (ShenandoahGCStateCheckRemove) {
806 // Unrealistic: remove all barrier fastpath checks.
807 } else if (ShenandoahGCStateCheckHotpatch) {
808 __ nop();
809 } else {
810 int bit_to_check = ShenandoahThreadLocalData::gc_state_to_fast_bit(test_state);
811 Address gc_state_fast(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
812 __ lbu(gcstate, gc_state_fast);
813 __ test_bit(gcstate, gcstate, bit_to_check);
814 __ bnez(gcstate, *slow_stub->entry());
815
816 // Fast path falls through here when the barrier is not needed.
817 __ bind(*slow_stub->continuation());
818 }
819 }
820
821 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr,
822 Register oldval, Register newval, Register tmp, bool exchange, bool maybe_null, bool narrow, bool weak) {
823 const Assembler::Aqrl acquire = needs_acquiring_load_reserved(node) ? Assembler::aq : Assembler::relaxed;
824 const Assembler::Aqrl release = Assembler::rl;
825
826 // Pre-barrier covers several things:
827 // a. Avoids false positives from CAS encountering to-space memory values.
828 // b. Satisfies the need for LRB for the CAE result.
829 // c. Records old value for the sake of SATB.
830 //
831 // (a) and (b) are covered because load barrier does memory location fixup.
832 // (c) is covered by KA on the current memory value.
833 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
834 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, Address(addr, 0), narrow, /* do_load: */ true);
835 char check = 0;
836 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
837 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
838 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for CAS/CAE");
839 ShenandoahBarrierStubC2::gc_state_check_c2(masm, t0, check, stub);
840 }
841
842 // Existing RISCV cmpxchg_oop already handles Shenandoah forwarded-value retry logic.
843 // It returns:
844 // - boolean 0/1 for CAS (!exchange)
845 // - loaded/current value for CAE (exchange)
846 ShenandoahBarrierSet::assembler()->cmpxchg_oop(masm, addr, oldval, newval, acquire, release, exchange /* is_cae */, res);
847
848 // Post-barrier deals with card updates.
849 card_barrier_c2(node, masm, Address(addr, 0));
850 }
851
852 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval,
853 Register newval, Register addr, Register tmp) {
854 const bool acquire = needs_acquiring_load_reserved(node);
855 const bool narrow = node->bottom_type()->isa_narrowoop();
856
857 // Pre-barrier covers several things:
858 // a. Satisfies the need for LRB for the GAS result.
859 // b. Records old value for the sake of SATB.
860 //
861 // (a) is covered because load barrier does memory location fixup.
862 // (b) is covered by KA on the current memory value.
863 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
864 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, Address(addr, 0), narrow, /* do_load: */ true);
865 char check = 0;
866 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
867 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
868 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for GAS");
869 ShenandoahBarrierStubC2::gc_state_check_c2(masm, t0, check, stub);
870 }
871
872 if (narrow) {
873 if (acquire) {
874 __ atomic_xchgalw(preval, newval, addr);
875 } else {
876 __ atomic_xchgw(preval, newval, addr);
877 }
878 } else {
879 if (acquire) {
880 __ atomic_xchgal(preval, newval, addr);
881 } else {
882 __ atomic_xchg(preval, newval, addr);
883 }
884 }
885
886 // Post-barrier deals with card updates.
887 card_barrier_c2(node, masm, Address(addr, 0));
888 }
889
890 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm, Address dst, bool dst_narrow,
891 Register src, bool src_narrow, Register tmp) {
892
893 // Pre-barrier: SATB / keep-alive on current value in memory.
894 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
895 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier(node), "Should not be required for stores");
896 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, dst, dst_narrow, /* do_load: */ true);
897 ShenandoahBarrierStubC2::gc_state_check_c2(masm, t0, ShenandoahHeap::MARKING, stub);
898 }
899
900 // Do the actual store
901 const bool is_volatile = node->has_trailing_membar();
902 if (dst_narrow) {
903 Register actual_src = src;
904 if (!src_narrow) {
905 assert(tmp != noreg, "need temp register");
906 __ mv(tmp, src);
907 if (ShenandoahBarrierStubC2::src_not_null(node)) {
908 __ encode_heap_oop_not_null(tmp, tmp);
909 } else {
910 __ encode_heap_oop(tmp, tmp);
911 }
912 actual_src = tmp;
913 }
914
915 if (is_volatile) {
916 __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
917 __ sw(actual_src, dst);
918 } else {
919 __ sw(actual_src, dst);
920 }
921 } else {
922 if (is_volatile) {
923 __ membar(MacroAssembler::StoreStore | MacroAssembler::LoadStore);
924 __ sd(src, dst);
925 } else {
926 __ sd(src, dst);
927 }
928 }
929
930 // Post-barrier: card updates.
931 card_barrier_c2(node, masm, dst);
932 }
933
934 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src) {
935 const bool acquire = node->memory_order() == MemNode::MemOrd::acquire;
936 const bool narrow = node->bottom_type()->isa_narrowoop();
937
938 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
939 if (narrow) {
940 __ lwu(dst, src);
941 if (acquire) {
942 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
943 }
944 } else {
945 __ ld(dst, src);
946 if (acquire) {
947 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
948 }
949 }
950
951 // Post-barrier: LRB / KA / weak-root processing.
952 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
953 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, dst, src, narrow, /* do_load: */ false);
954 char check = 0;
955 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
956 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
957 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
958 ShenandoahBarrierStubC2::gc_state_check_c2(masm, t0, check, stub);
959 }
960 }
961
962 void ShenandoahBarrierSetAssembler::card_barrier_c2(const MachNode* node, MacroAssembler* masm, Address address) {
963 if (ShenandoahSkipBarriers || (node->barrier_data() & ShenandoahBitCardMark) == 0) {
964 return;
965 }
966
967 assert(CardTable::dirty_card_val() == 0, "must be");
968 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
969
970 // t1 = effective address
971 __ la(t1, address);
972
973 // t1 = card index
974 __ srli(t1, t1, CardTable::card_shift());
975
976 // t0 = card table base
977 Address curr_ct_holder_addr(xthread, in_bytes(ShenandoahThreadLocalData::card_table_offset()));
978 __ ld(t0, curr_ct_holder_addr);
979
980 // t1 = &card_table[card_index]
981 __ add(t1, t1, t0);
982
983 if (UseCondCardMark) {
984 Label already_dirty;
985 __ lbu(t0, Address(t1));
986 __ beqz(t0, already_dirty);
987 __ sb(zr, Address(t1));
988 __ bind(already_dirty);
989 } else {
990 __ sb(zr, Address(t1));
991 }
992 }
993
994 #undef __
995 #define __ masm.
996
997 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
998 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
999
1000 // Stub entry
1001 __ bind(*BarrierStubC2::entry());
1002
1003 // If needed, perform the load here so stub logic sees the current oop value.
1004 if (_do_load) {
1005 __ load_heap_oop(_obj, _addr, noreg, noreg, AS_RAW);
1006 } else if (_narrow) {
1007 // Decode narrow oop before barrier processing.
1008 if (_maybe_null) {
1009 __ decode_heap_oop(_obj, _obj);
1010 } else {
1011 __ decode_heap_oop_not_null(_obj, _obj);
1012 }
1013 }
1014
1015 if (_do_load || _maybe_null) {
1016 __ beqz(_obj, *continuation());
1017 }
1018
1019 keepalive(&masm, _obj, t0, t1);
1020
1021 lrb(&masm, _obj, _addr, noreg);
1022
1023 // If object is narrow, we need to encode it before exiting.
1024 // For encoding, dst can only turn null if we are dealing with weak loads.
1025 // Otherwise, we have already null-checked. We can skip all this if we performed
1026 // the load ourselves, which means the value is not used by caller.
1027 if (_narrow && !_do_load) {
1028 if (_needs_load_ref_weak_barrier) {
1029 __ encode_heap_oop(_obj, _obj);
1030 } else {
1031 __ encode_heap_oop_not_null(_obj, _obj);
1032 }
1033 }
1034
1035 // Go back to fast path
1036 __ j(*continuation());
1037 }
1038
1039 #undef __
1040 #define __ masm->
1041
1042 void ShenandoahBarrierStubC2::keepalive(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
1043 Address index(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1044 Address buffer(xthread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1045 Label L_runtime;
1046 Label L_done;
1047
1048 // The node doesn't even need keepalive barrier, just don't check anything else
1049 if (!_needs_keep_alive_barrier) {
1050 return;
1051 }
1052
1053 // If both LRB and KeepAlive barriers are required (rare), do a runtime check
1054 // for enabled barrier.
1055 if (_needs_load_ref_barrier) {
1056 Address gcs_addr(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1057 __ lbu(t0, gcs_addr);
1058 __ test_bit(t0, t0, ShenandoahHeap::MARKING_BITPOS);
1059 __ beqz(t0, L_done);
1060 }
1061
1062 // If the queue is full, go to runtime.
1063 __ ld(tmp1, index);
1064 __ beqz(tmp1, L_runtime);
1065
1066 // Push into SATB queue.
1067 __ subi(tmp1, tmp1, wordSize);
1068 __ sd(tmp1, index);
1069 __ ld(tmp2, buffer);
1070 __ add(tmp1, tmp1, tmp2);
1071 __ sd(obj, Address(tmp1, 0));
1072 __ j(L_done);
1073
1074 // Runtime call
1075 __ bind(L_runtime);
1076
1077 preserve(obj);
1078 {
1079 SaveLiveRegisters save_registers(masm, this);
1080 __ mv(c_rarg0, obj);
1081 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::write_barrier_pre), c_rarg0);
1082 }
1083
1084 __ bind(L_done);
1085 }
1086
1087 void ShenandoahBarrierStubC2::lrb(MacroAssembler* masm, Register obj, Address addr, Register tmp) {
1088 Label L_done;
1089
1090 // The node doesn't even need LRB barrier, just don't check anything else
1091 if (!_needs_load_ref_barrier) {
1092 return;
1093 }
1094
1095 if ((_node->barrier_data() & ShenandoahBitStrong) != 0) {
1096 // If both LRB and KeepAlive barriers are required (rare), do a runtime
1097 // check for enabled barrier.
1098 if (_needs_keep_alive_barrier) {
1099 Address gcs_addr(xthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1100 __ lbu(t0, gcs_addr);
1101
1102 if (_needs_load_ref_weak_barrier) {
1103 __ srli(t1, t0, ShenandoahHeap::WEAK_ROOTS_BITPOS);
1104 __ orr(t0, t0, t1);
1105 }
1106
1107 __ test_bit(t0, t0, ShenandoahHeap::HAS_FORWARDED_BITPOS);
1108 __ beqz(t0, L_done);
1109 }
1110
1111 // Weak/phantom loads always need to go to runtime. For strong refs we
1112 // check if the object in cset, if they are not, then we are done with LRB.
1113 __ mv(t1, ShenandoahHeap::in_cset_fast_test_addr());
1114 __ srli(t0, obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1115 __ add(t1, t1, t0);
1116 __ lbu(t1, Address(t1));
1117 __ beqz(t1, L_done);
1118 }
1119
1120 dont_preserve(obj);
1121 {
1122 SaveLiveRegisters save_registers(masm, this);
1123
1124 // Runtime call wants:
1125 // c_rarg0 <- obj
1126 // c_rarg1 <- lea(addr)
1127 if (c_rarg0 == obj) {
1128 __ la(c_rarg1, addr);
1129 } else if (c_rarg1 == obj) {
1130 // Set up arguments in reverse, and then flip them
1131 __ la(c_rarg0, addr);
1132 // flip them
1133 __ mv(t0, c_rarg0);
1134 __ mv(c_rarg0, c_rarg1);
1135 __ mv(c_rarg1, t0);
1136 } else {
1137 assert_different_registers(c_rarg1, obj);
1138 __ la(c_rarg1, addr);
1139 __ mv(c_rarg0, obj);
1140 }
1141
1142 // Get address of runtime LRB entry and call it
1143 __ rt_call(lrb_runtime_entry_addr());
1144
1145 // If we loaded the object in the stub it means we don't need to return it
1146 // to fastpath, so no need to make this mov.
1147 if (!_do_load) {
1148 __ mv(obj, x10);
1149 }
1150 }
1151
1152 __ bind(L_done);
1153 }
1154
1155 Label* ShenandoahBarrierStubC2::entry() {
1156 return BarrierStubC2::entry();
1157 }
1158
1159 ShenandoahBarrierStubC2::ShenandoahBarrierStubC2(const MachNode* node, Register obj, Address addr, bool narrow, bool do_load, int offset) : BarrierStubC2(node),
1160 _obj(obj),
1161 _addr(addr),
1162 _do_load(do_load),
1163 _narrow(narrow),
1164 _maybe_null(!src_not_null(node)),
1165 _needs_load_ref_barrier(needs_load_ref_barrier(node)),
1166 _needs_load_ref_weak_barrier(needs_load_ref_barrier_weak(node)),
1167 _needs_keep_alive_barrier(needs_keep_alive_barrier(node)),
1168 _fastpath_branch_offset(),
1169 _test_and_branch_reachable(),
1170 _skip_trampoline(),
1171 _test_and_branch_reachable_entry() {
1172
1173 ShenandoahBarrierStubC2(node, obj, addr, narrow, do_load);
1174 }
1175 #endif // COMPILER2
|