25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
49 Register src, Register dst, Register count) {
50
51 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
52
53 if (is_reference_type(type)) {
54 if (ShenandoahCardBarrier) {
55 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
56 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
57 bool obj_int = (type == T_OBJECT) && UseCompressedOops;
58
59 // We need to save the original element count because the array copy stub
60 // will destroy the value and we need it for the card marking barrier.
61 if (!checkcast) {
62 if (!obj_int) {
63 // Save count for barrier
64 __ movptr(r11, count);
884 assert(!is_native, "weak must not be called off-heap");
885 if (UseCompressedOops) {
886 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
887 } else {
888 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
889 }
890 } else {
891 assert(is_phantom, "only remaining strength");
892 assert(is_native, "phantom must only be called off-heap");
893 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
894 }
895
896 __ restore_live_registers_except_rax(true);
897
898 __ epilogue();
899 }
900
901 #undef __
902
903 #endif // COMPILER1
|
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #endif
48
49 #define __ masm->
50
51 void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, DecoratorSet decorators, BasicType type,
52 Register src, Register dst, Register count) {
53
54 bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
55
56 if (is_reference_type(type)) {
57 if (ShenandoahCardBarrier) {
58 bool checkcast = (decorators & ARRAYCOPY_CHECKCAST) != 0;
59 bool disjoint = (decorators & ARRAYCOPY_DISJOINT) != 0;
60 bool obj_int = (type == T_OBJECT) && UseCompressedOops;
61
62 // We need to save the original element count because the array copy stub
63 // will destroy the value and we need it for the card marking barrier.
64 if (!checkcast) {
65 if (!obj_int) {
66 // Save count for barrier
67 __ movptr(r11, count);
887 assert(!is_native, "weak must not be called off-heap");
888 if (UseCompressedOops) {
889 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
890 } else {
891 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
892 }
893 } else {
894 assert(is_phantom, "only remaining strength");
895 assert(is_native, "phantom must only be called off-heap");
896 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
897 }
898
899 __ restore_live_registers_except_rax(true);
900
901 __ epilogue();
902 }
903
904 #undef __
905
906 #endif // COMPILER1
907
908 #ifdef COMPILER2
909
910 #undef __
911 #define __ masm->
912
913 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, bool narrow) {
914 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
915 if (narrow) {
916 __ movl(dst, src);
917 } else {
918 __ movq(dst, src);
919 }
920
921 ShenandoahBarrierStubC2::load_post(masm, node, dst, src, noreg, noreg, narrow);
922 }
923
924 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
925 Address dst, bool dst_narrow,
926 Register src, bool src_narrow,
927 Register tmp) {
928
929 ShenandoahBarrierStubC2::store_pre(masm, node, tmp, dst, noreg, noreg, dst_narrow);
930
931 // Need to encode into tmp, because we cannot clobber src.
932 if (dst_narrow && !src_narrow) {
933 __ movq(tmp, src);
934 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
935 __ encode_heap_oop(tmp);
936 } else {
937 __ encode_heap_oop_not_null(tmp);
938 }
939 src = tmp;
940 }
941
942 // Do the actual store
943 if (dst_narrow) {
944 __ movl(dst, src);
945 } else {
946 __ movq(dst, src);
947 }
948
949 ShenandoahBarrierStubC2::store_post(masm, node, dst, tmp, noreg);
950 }
951
952 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm,
953 Register res, Address addr,
954 Register oldval, Register newval, Register tmp,
955 bool narrow) {
956
957 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
958
959 // Oldval and newval can be in the same register, but all other registers should be
960 // distinct for extra safety, as we shuffle register values around.
961 assert_different_registers(oldval, tmp, addr.base(), addr.index());
962 assert_different_registers(newval, tmp, addr.base(), addr.index());
963
964 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp, addr, noreg, noreg, narrow);
965
966 // CAS!
967 __ lock();
968 if (narrow) {
969 __ cmpxchgl(newval, addr);
970 } else {
971 __ cmpxchgptr(newval, addr);
972 }
973
974 // If we need a boolean result out of CAS, set the flag appropriately and promote the result.
975 if (res != noreg) {
976 __ setcc(Assembler::equal, res);
977 }
978
979 ShenandoahBarrierStubC2::load_store_post(masm, node, addr, tmp, noreg);
980 }
981
982 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register newval, Address addr, Register tmp, bool narrow) {
983 assert_different_registers(newval, tmp, addr.base(), addr.index());
984
985 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp, addr, noreg, noreg, narrow);
986
987 if (narrow) {
988 __ xchgl(newval, addr);
989 } else {
990 __ xchgq(newval, addr);
991 }
992
993 ShenandoahBarrierStubC2::load_store_post(masm, node, addr, tmp, noreg);
994 }
995
996 #undef __
997 #define __ masm.
998
999 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address addr, Register tmp1, Register tmp2) {
1000 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1001
1002 __ lea(tmp1, addr);
1003 __ shrptr(tmp1, CardTable::card_shift());
1004 __ addptr(tmp1, Address(r15_thread, in_bytes(ShenandoahThreadLocalData::card_table_offset())));
1005 Address card_address(tmp1, 0);
1006
1007 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
1008 Label L_done;
1009 if (UseCondCardMark) {
1010 __ cmpb(card_address, 0);
1011 __ jccb(Assembler::equal, L_done);
1012 }
1013 if (UseCompressedOops && CompressedOops::base() == nullptr) {
1014 __ movb(card_address, r12);
1015 } else {
1016 __ movb(card_address, 0);
1017 }
1018 __ bind(L_done);
1019 }
1020
1021 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
1022 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1023
1024 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)));
1025 __ cmpb(gc_state_fast, 0);
1026 __ jcc(Assembler::notEqual, *entry());
1027 __ bind(*continuation());
1028 }
1029
1030 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1031 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1032
1033 __ align(InteriorEntryAlignment);
1034 __ bind(*entry());
1035
1036 // If we need to load ourselves, do it here.
1037 if (_do_load) {
1038 if (_narrow) {
1039 __ movl(_obj, _addr);
1040 } else {
1041 __ movq(_obj, _addr);
1042 }
1043 }
1044
1045 // If the object is null, there is no point in applying barriers.
1046 if (_narrow) {
1047 __ testl(_obj, _obj);
1048 } else {
1049 __ testq(_obj, _obj);
1050 }
1051 __ jcc(Assembler::zero, *continuation());
1052
1053 // We need to make sure that loads done by callers survive across slow-path calls.
1054 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1055 if (!_do_load || (_needs_keep_alive_barrier && _needs_load_ref_barrier)) {
1056 preserve(_obj);
1057 }
1058
1059 // Go for barriers. If both KA and LRB are needed (rare), do additional gc-state
1060 // checks to verify which one is currently needed. Note that KA and LRB are *not*
1061 // exclusive, because we can have an overlapping marking/evac in generational mode.
1062 if (_needs_keep_alive_barrier && _needs_load_ref_barrier) {
1063 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1064
1065 Label L_skip_keepalive;
1066 __ testb(gc_state, ShenandoahHeap::MARKING);
1067 __ jcc(Assembler::zero, L_skip_keepalive);
1068 keepalive(masm, nullptr);
1069 __ bind(L_skip_keepalive);
1070
1071 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0));
1072 __ jcc(Assembler::zero, *continuation());
1073 lrb(masm);
1074 } else if (_needs_keep_alive_barrier) {
1075 keepalive(masm, continuation());
1076 } else if (_needs_load_ref_barrier) {
1077 lrb(masm);
1078 } else {
1079 ShouldNotReachHere();
1080 }
1081 }
1082
1083 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1084 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1085 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1086
1087 Label L_pop_and_slow;
1088
1089 // Need temp to work, allocate one now.
1090 bool tmp_live;
1091 Register tmp = select_temp_register(tmp_live);
1092 if (tmp_live) {
1093 __ push(tmp);
1094 }
1095
1096 // Fast-path: put object into buffer.
1097 // If buffer is already full, go slow.
1098 __ movptr(tmp, index);
1099 __ subptr(tmp, wordSize);
1100 __ jccb(Assembler::below, L_pop_and_slow);
1101 __ movptr(index, tmp);
1102 __ addptr(tmp, buffer);
1103
1104 // Store the object in queue.
1105 // If object is narrow, we need to decode it before inserting.
1106 // We can skip the re-encoding if we know that object is not preserved.
1107 if (_narrow) {
1108 __ decode_heap_oop_not_null(_obj);
1109 }
1110 __ movptr(Address(tmp, 0), _obj);
1111 if (_narrow && is_preserved(_obj)) {
1112 __ encode_heap_oop_not_null(_obj);
1113 }
1114
1115 // Fast-path exits here.
1116 if (tmp_live) {
1117 __ pop(tmp);
1118 }
1119
1120 Label L_fallthrough;
1121 if (L_done != nullptr) {
1122 __ jmp(*L_done);
1123 } else {
1124 __ jmp(L_fallthrough);
1125 }
1126
1127 // Slow-path: call runtime to handle.
1128 // Need to pop tmp immediately for stack to remain aligned.
1129 __ bind(L_pop_and_slow);
1130 if (tmp_live) {
1131 __ pop(tmp);
1132 }
1133 {
1134 SaveLiveRegisters slr(&masm, this);
1135
1136 // Shuffle in the arguments. The end result should be:
1137 // c_rarg0 <-- obj
1138 if (c_rarg0 != _obj) {
1139 __ mov(c_rarg0, _obj);
1140 }
1141
1142 // Go to runtime and handle the rest there.
1143 __ call(RuntimeAddress(keepalive_runtime_entry_addr()));
1144 }
1145 if (L_done != nullptr) {
1146 __ jmp(*L_done);
1147 } else {
1148 __ bind(L_fallthrough);
1149 }
1150 }
1151
1152 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1153 Label L_pop_and_slow, L_slow;
1154
1155 // If weak references are being processed, weak/phantom loads need to go slow,
1156 // regardless of their cset status.
1157 if (_needs_load_ref_weak_barrier) {
1158 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1159 __ testb(gc_state, ShenandoahHeap::WEAK_ROOTS);
1160 __ jccb(Assembler::notZero, L_slow);
1161 }
1162
1163 // Need temp to work, allocate one now.
1164 bool tmp_live;
1165 Register tmp = select_temp_register(tmp_live);
1166 if (tmp_live) {
1167 __ push(tmp);
1168 }
1169
1170 // Compute the cset bitmap index
1171 if (_narrow) {
1172 __ decode_heap_oop_not_null(tmp, _obj);
1173 } else {
1174 __ movptr(tmp, _obj);
1175 }
1176 __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1177
1178 Address cset_addr_arg;
1179 intptr_t cset_addr = reinterpret_cast<intptr_t>(ShenandoahHeap::in_cset_fast_test_addr());
1180 if ((cset_addr >> 3) < INT32_MAX) {
1181 // Cset bitmap is at easily encodeable address. Just use it as offset.
1182 assert(is_aligned(cset_addr, 8), "Sanity");
1183 cset_addr_arg = Address(tmp, cset_addr >> 3, Address::times_8);
1184 } else {
1185 // Cset bitmap is way further than our encoding limit. Add its address fully.
1186 bool tmp2_live;
1187 Register tmp2 = select_temp_register(tmp2_live, /* skip_reg1 = */ tmp);
1188 if (tmp2_live) {
1189 __ push(tmp2);
1190 }
1191 __ movptr(tmp2, cset_addr);
1192 __ addptr(tmp, tmp2);
1193 if (tmp2_live) {
1194 __ pop(tmp2);
1195 }
1196 cset_addr_arg = Address(tmp, 0);
1197 }
1198
1199 // Cset-check. Fall-through to slow if in collection set.
1200 __ cmpb(cset_addr_arg, 0);
1201 if (tmp_live) {
1202 __ jccb(Assembler::notEqual, L_pop_and_slow);
1203 __ pop(tmp);
1204 __ jmp(*continuation());
1205 } else {
1206 // Nothing else to do, jump back
1207 __ jcc(Assembler::equal, *continuation());
1208 }
1209
1210 // Slow path
1211 __ bind(L_pop_and_slow);
1212 // Need to pop tmp immediately for stack to remain aligned.
1213 if (tmp_live) {
1214 __ pop(tmp);
1215 }
1216 __ bind(L_slow);
1217
1218 // Obj is the result, need to temporarily stop preserving it.
1219 bool is_obj_preserved = is_preserved(_obj);
1220 if (is_obj_preserved) {
1221 dont_preserve(_obj);
1222 }
1223 {
1224 SaveLiveRegisters slr(&masm, this);
1225
1226 assert_different_registers(rax, c_rarg0, c_rarg1);
1227
1228 // Shuffle in the arguments. The end result should be:
1229 // c_rarg0 <-- obj
1230 // c_rarg1 <-- lea(addr)
1231 if (_obj == c_rarg0) {
1232 __ lea(c_rarg1, _addr);
1233 } else if (_obj == c_rarg1) {
1234 // Set up arguments in reverse, and then flip them
1235 __ lea(c_rarg0, _addr);
1236 __ xchgptr(c_rarg0, c_rarg1);
1237 } else {
1238 assert_different_registers(_obj, c_rarg0, c_rarg1);
1239 __ lea(c_rarg1, _addr);
1240 __ movptr(c_rarg0, _obj);
1241 }
1242
1243 // Go to runtime and handle the rest there.
1244 __ call(RuntimeAddress(lrb_runtime_entry_addr()));
1245
1246 // Save the result where needed.
1247 if (_narrow) {
1248 __ movl(_obj, rax);
1249 } else if (_obj != rax) {
1250 __ movptr(_obj, rax);
1251 }
1252 }
1253 if (is_obj_preserved) {
1254 preserve(_obj);
1255 }
1256
1257 __ jmp(*continuation());
1258 }
1259
1260 int ShenandoahBarrierStubC2::available_gp_registers() {
1261 return Register::available_gp_registers();
1262 }
1263
1264 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1265 return r == rsp || r == rbp || r == r12_heapbase || r == r15_thread;
1266 }
1267
1268 void ShenandoahBarrierStubC2::post_init() {
1269 // Do nothing.
1270 }
1271
1272 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1273 Unimplemented(); // Not used.
1274 }
1275
1276 #endif // COMPILER2
|