25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45
46 #define __ masm->
47
48 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
49 if (handle_gpr) {
50 __ push_IU_state();
51 }
52
53 if (handle_fp) {
54 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
55 assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call");
56
57 const int xmm_size = wordSize * 2;
58 __ subptr(rsp, xmm_size * 8);
59 __ movdbl(Address(rsp, xmm_size * 0), xmm0);
60 __ movdbl(Address(rsp, xmm_size * 1), xmm1);
61 __ movdbl(Address(rsp, xmm_size * 2), xmm2);
62 __ movdbl(Address(rsp, xmm_size * 3), xmm3);
63 __ movdbl(Address(rsp, xmm_size * 4), xmm4);
64 __ movdbl(Address(rsp, xmm_size * 5), xmm5);
996 assert(!is_native, "weak must not be called off-heap");
997 if (UseCompressedOops) {
998 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
999 } else {
1000 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
1001 }
1002 } else {
1003 assert(is_phantom, "only remaining strength");
1004 assert(is_native, "phantom must only be called off-heap");
1005 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
1006 }
1007
1008 __ restore_live_registers_except_rax(true);
1009
1010 __ epilogue();
1011 }
1012
1013 #undef __
1014
1015 #endif // COMPILER1
|
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #endif
48
49 #define __ masm->
50
51 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
52 if (handle_gpr) {
53 __ push_IU_state();
54 }
55
56 if (handle_fp) {
57 // Some paths can be reached from the c2i adapter with live fp arguments in registers.
58 assert(Argument::n_float_register_parameters_j == 8, "8 fp registers to save at java call");
59
60 const int xmm_size = wordSize * 2;
61 __ subptr(rsp, xmm_size * 8);
62 __ movdbl(Address(rsp, xmm_size * 0), xmm0);
63 __ movdbl(Address(rsp, xmm_size * 1), xmm1);
64 __ movdbl(Address(rsp, xmm_size * 2), xmm2);
65 __ movdbl(Address(rsp, xmm_size * 3), xmm3);
66 __ movdbl(Address(rsp, xmm_size * 4), xmm4);
67 __ movdbl(Address(rsp, xmm_size * 5), xmm5);
999 assert(!is_native, "weak must not be called off-heap");
1000 if (UseCompressedOops) {
1001 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak_narrow), c_rarg0, c_rarg1);
1002 } else {
1003 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
1004 }
1005 } else {
1006 assert(is_phantom, "only remaining strength");
1007 assert(is_native, "phantom must only be called off-heap");
1008 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
1009 }
1010
1011 __ restore_live_registers_except_rax(true);
1012
1013 __ epilogue();
1014 }
1015
1016 #undef __
1017
1018 #endif // COMPILER1
1019
1020 #ifdef COMPILER2
1021 #undef __
1022 #define __ masm->
1023
1024 bool ShenandoahBarrierStubC2::push_save_register_if_live(MacroAssembler* masm, Register reg) {
1025 if (is_live(reg)) {
1026 push_save_register(masm, reg);
1027 return true;
1028 } else {
1029 return false;
1030 }
1031 }
1032
1033 void ShenandoahBarrierStubC2::push_save_register(MacroAssembler* masm, Register reg) {
1034 __ movptr(Address(rsp, push_save_slot()), reg);
1035 }
1036
1037 void ShenandoahBarrierStubC2::pop_save_register(MacroAssembler* masm, Register reg) {
1038 __ movptr(reg, Address(rsp, pop_save_slot()));
1039 }
1040
1041 bool ShenandoahBarrierStubC2::is_live(Register reg) {
1042 // TODO: Precompute the generic register map for faster lookups.
1043 RegMaskIterator rmi(preserve_set());
1044 while (rmi.has_next()) {
1045 const OptoReg::Name opto_reg = rmi.next();
1046 const VMReg vm_reg = OptoReg::as_VMReg(opto_reg);
1047 if (vm_reg->is_Register()) {
1048 if (reg == vm_reg->as_Register()) {
1049 return true;
1050 }
1051 } else if (vm_reg->is_KRegister()) {
1052 // Do not care, skip.
1053 } else if (vm_reg->is_XMMRegister()) {
1054 // Do not care, skip.
1055 } else {
1056 fatal("Unexpected register type");
1057 }
1058 }
1059 return false;
1060 }
1061
1062 Register ShenandoahBarrierStubC2::select_temp_register(bool& selected_live, Address addr, Register reg1) {
1063 Register tmp = noreg;
1064 Register fallback_live = noreg;
1065
1066 // Try to select non-live first:
1067 for (int i = 0; i < Register::available_gp_registers(); i++) {
1068 Register r = as_Register(i);
1069 if (r != rsp && r != rbp && r != r12_heapbase && r != r15_thread &&
1070 r != reg1 && r != addr.base() && r != addr.index()) {
1071 if (!is_live(r)) {
1072 tmp = r;
1073 break;
1074 } else if (fallback_live == noreg) {
1075 fallback_live = r;
1076 }
1077 }
1078 }
1079
1080 // If we could not find a non-live register, select the live fallback:
1081 if (tmp == noreg) {
1082 tmp = fallback_live;
1083 selected_live = true;
1084 } else {
1085 selected_live = false;
1086 }
1087
1088 assert(tmp != noreg, "successfully selected");
1089 assert_different_registers(tmp, reg1);
1090 assert_different_registers(tmp, addr.base());
1091 assert_different_registers(tmp, addr.index());
1092 return tmp;
1093 }
1094
1095 void ShenandoahBarrierSetAssembler::gc_state_check_c2(MacroAssembler* masm, const char test_state, BarrierStubC2* slow_stub) {
1096 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
1097
1098 if (ShenandoahGCStateCheckRemove) {
1099 // Unrealistic: remove all barrier fastpath checks.
1100 } else if (ShenandoahGCStateCheckHotpatch) {
1101 // In the ideal world, we would hot-patch the branch to slow stub with a single
1102 // (unconditional) jump or nop, based on our current GC state. Unconditional jump
1103 // to near target within the nmethod (at 32-bit offset) takes 5 bytes.
1104 __ nop(5);
1105 } else {
1106 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
1107 __ testb(gc_state_fast, ShenandoahThreadLocalData::gc_state_to_fast(test_state));
1108 __ jcc(Assembler::notZero, *slow_stub->entry());
1109 __ bind(*slow_stub->continuation());
1110 }
1111 }
1112
1113 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, bool narrow) {
1114 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
1115 if (narrow) {
1116 __ movl(dst, src);
1117 } else {
1118 __ movq(dst, src);
1119 }
1120
1121 // Post-barrier: LRB
1122 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
1123 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, dst, src, narrow, false);
1124 char check = 0;
1125 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
1126 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
1127 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
1128 gc_state_check_c2(masm, check, stub);
1129 }
1130 }
1131
1132 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
1133 Address dst, bool dst_narrow,
1134 Register src, bool src_narrow,
1135 Register tmp) {
1136
1137 // Pre-barrier: SATB, keep-alive the current memory value.
1138 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
1139 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier(node), "Should not be required for stores");
1140 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, dst, dst_narrow, true);
1141 gc_state_check_c2(masm, ShenandoahHeap::MARKING, stub);
1142 }
1143
1144 // Need to encode into tmp, because we cannot clobber src.
1145 // TODO: Maybe there is a matcher way to test that src is unused after this?
1146 if (dst_narrow && !src_narrow) {
1147 __ movq(tmp, src);
1148 if (ShenandoahBarrierStubC2::src_not_null(node)) {
1149 __ encode_heap_oop_not_null(tmp);
1150 } else {
1151 __ encode_heap_oop(tmp);
1152 }
1153 src = tmp;
1154 }
1155
1156 // Do the actual store
1157 if (dst_narrow) {
1158 __ movl(dst, src);
1159 } else {
1160 __ movq(dst, src);
1161 }
1162
1163 // Post-barrier: card updates.
1164 if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
1165 card_barrier_c2(masm, dst, tmp);
1166 }
1167 }
1168
1169 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm,
1170 Register res, Address addr,
1171 Register oldval, Register newval, Register tmp,
1172 bool narrow) {
1173
1174 assert(oldval == rax, "must be in rax for implicit use in cmpxchg");
1175
1176 // Oldval and newval can be in the same register, but all other registers should be
1177 // distinct for extra safety, as we shuffle register values around.
1178 assert_different_registers(oldval, tmp, addr.base(), addr.index());
1179 assert_different_registers(newval, tmp, addr.base(), addr.index());
1180
1181 // Pre-barrier covers several things:
1182 // a. Avoids false positives from CAS encountering to-space memory values.
1183 // b. Satisfies the need for LRB for the CAE result.
1184 // c. Records old value for the sake of SATB.
1185 //
1186 // (a) and (b) are covered because load barrier does memory location fixup.
1187 // (c) is covered by KA on the current memory value.
1188 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
1189 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, addr, narrow, true);
1190 char check = 0;
1191 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
1192 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
1193 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for CAS");
1194 gc_state_check_c2(masm, check, stub);
1195 }
1196
1197 // CAS!
1198 __ lock();
1199 if (narrow) {
1200 __ cmpxchgl(newval, addr);
1201 } else {
1202 __ cmpxchgptr(newval, addr);
1203 }
1204
1205 // If we need a boolean result out of CAS, set the flag appropriately and promote the result.
1206 if (res != noreg) {
1207 __ setcc(Assembler::equal, res);
1208 }
1209
1210 // Post-barrier deals with card updates.
1211 if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
1212 card_barrier_c2(masm, addr, tmp);
1213 }
1214 }
1215
1216 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register newval, Address addr, Register tmp, bool narrow) {
1217 assert_different_registers(newval, tmp, addr.base(), addr.index());
1218
1219 // Pre-barrier covers several things:
1220 // a. Satisfies the need for LRB for the GAS result.
1221 // b. Records old value for the sake of SATB.
1222 //
1223 // (a) is covered because load barrier does memory location fixup.
1224 // (b) is covered by KA on the current memory value.
1225 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
1226 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, tmp, addr, narrow, true);
1227 char check = 0;
1228 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
1229 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
1230 assert(!ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node), "Not supported for GAS");
1231 gc_state_check_c2(masm, check, stub);
1232 }
1233
1234 if (narrow) {
1235 __ xchgl(newval, addr);
1236 } else {
1237 __ xchgq(newval, addr);
1238 }
1239
1240 // Post-barrier deals with card updates.
1241 if (ShenandoahBarrierStubC2::needs_card_barrier(node)) {
1242 card_barrier_c2(masm, addr, tmp);
1243 }
1244 }
1245
1246 void ShenandoahBarrierSetAssembler::card_barrier_c2(MacroAssembler* masm, Address dst, Register tmp) {
1247 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
1248
1249 // TODO: Might be a good place to implement some filters here.
1250 // For example, G1 only flips card marks for stores within a single region.
1251
1252 __ lea(tmp, dst);
1253 __ shrptr(tmp, CardTable::card_shift());
1254 __ addptr(tmp, Address(r15_thread, in_bytes(ShenandoahThreadLocalData::card_table_offset())));
1255 Address card_address(tmp, 0);
1256
1257 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
1258 Label L_done;
1259 if (UseCondCardMark) {
1260 __ cmpb(card_address, 0);
1261 __ jccb(Assembler::equal, L_done);
1262 }
1263 if (UseCompressedOops && CompressedOops::base() == nullptr) {
1264 __ movb(card_address, r12);
1265 } else {
1266 __ movb(card_address, 0);
1267 }
1268 __ bind(L_done);
1269 }
1270
1271 void ShenandoahBarrierStubC2::keepalive(MacroAssembler* masm, Register obj, Register tmp1, Register tmp2) {
1272 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1273 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1274
1275 Label L_fast, L_done;
1276
1277 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1278 if (_needs_load_ref_barrier) {
1279 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1280 __ testb(gc_state, ShenandoahHeap::MARKING);
1281 __ jccb(Assembler::zero, L_done);
1282 }
1283
1284 // Check if buffer is already full. Go slow, if so.
1285 __ movptr(tmp1, index);
1286 __ testptr(tmp1, tmp1);
1287 __ jccb(Assembler::notZero, L_fast);
1288
1289 // Slow-path: call runtime to handle.
1290 {
1291 // Shuffle in the arguments. The end result should be:
1292 // c_rarg0 <-- obj
1293 //
1294 // Save clobbered registers before overwriting them.
1295 bool clobbered_c_rarg0 = false;
1296 if (c_rarg0 != obj) {
1297 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1298 __ mov(c_rarg0, obj);
1299 }
1300
1301 // Go to runtime stub and handle the rest there.
1302 __ call(RuntimeAddress(keepalive_runtime_entry_addr()));
1303
1304 // Restore the clobbered registers.
1305 if (clobbered_c_rarg0) {
1306 pop_save_register(masm, c_rarg0);
1307 }
1308 __ jmpb(L_done);
1309 }
1310
1311 // Fast-path: put object into buffer.
1312 __ bind(L_fast);
1313 __ subptr(tmp1, wordSize);
1314 __ movptr(index, tmp1);
1315 __ addptr(tmp1, buffer);
1316 __ movptr(Address(tmp1, 0), obj);
1317
1318 __ bind(L_done);
1319 }
1320
1321 void ShenandoahBarrierStubC2::lrb(MacroAssembler* masm, Register obj, Address addr, Register tmp) {
1322 Label L_done;
1323
1324 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1325 if (_needs_keep_alive_barrier) {
1326 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1327 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0));
1328 __ jccb(Assembler::zero, L_done);
1329 }
1330
1331 // Weak/phantom loads are handled in slow path.
1332 if (!_needs_load_ref_weak_barrier) {
1333 // Compute the cset bitmap index
1334 __ movptr(tmp, obj);
1335 __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1336
1337 // If cset address is in good spot to just use it as offset. It almost always is.
1338 Address cset_addr_arg;
1339 intptr_t cset_addr = (intptr_t) ShenandoahHeap::in_cset_fast_test_addr();
1340 if ((cset_addr >> 3) < INT32_MAX) {
1341 assert(is_aligned(cset_addr, 8), "Sanity");
1342 cset_addr_arg = Address(tmp, checked_cast<int>(cset_addr >> 3), Address::times_8);
1343 } else {
1344 __ addptr(tmp, cset_addr);
1345 cset_addr_arg = Address(tmp, 0);
1346 }
1347
1348 // Cset-check. Fall-through to slow if in collection set.
1349 __ cmpb(cset_addr_arg, 0);
1350 __ jccb(Assembler::equal, L_done);
1351 }
1352
1353 // Slow path
1354 {
1355 assert_different_registers(rax, c_rarg0, c_rarg1);
1356
1357 // Shuffle in the arguments. The end result should be:
1358 // c_rarg0 <-- obj
1359 // c_rarg1 <-- lea(addr)
1360 //
1361 // Save clobbered registers before overwriting them, unless they
1362 // carry obj, which would be overwritten on return.
1363 bool clobbered_c_rarg0 = false;
1364 bool clobbered_c_rarg1 = false;
1365 bool clobbered_rax = false;
1366
1367 if (obj == c_rarg0) {
1368 clobbered_c_rarg1 = push_save_register_if_live(masm, c_rarg1);
1369 __ lea(c_rarg1, addr);
1370 } else if (obj == c_rarg1) {
1371 // Set up arguments in reverse, and then flip them
1372 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1373 __ lea(c_rarg0, addr);
1374 __ xchgptr(c_rarg0, c_rarg1);
1375 } else {
1376 assert_different_registers(obj, c_rarg0, c_rarg1);
1377 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1378 clobbered_c_rarg1 = push_save_register_if_live(masm, c_rarg1);
1379 __ lea(c_rarg1, addr);
1380 __ movptr(c_rarg0, obj);
1381 }
1382 if (obj != rax) {
1383 clobbered_rax = push_save_register_if_live(masm, rax);
1384 }
1385
1386 // Go to runtime stub and handle the rest there.
1387 __ call(RuntimeAddress(lrb_runtime_entry_addr()));
1388
1389 // Save the result where needed and restore the clobbered registers.
1390 if (obj != rax) {
1391 __ movptr(obj, rax);
1392 }
1393 if (clobbered_rax) {
1394 pop_save_register(masm, rax);
1395 }
1396 if (clobbered_c_rarg1) {
1397 pop_save_register(masm, c_rarg1);
1398 }
1399 if (clobbered_c_rarg0) {
1400 pop_save_register(masm, c_rarg0);
1401 }
1402 }
1403
1404 __ bind(L_done);
1405 }
1406
1407 #undef __
1408 #define __ masm.
1409
1410 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1411 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1412
1413 __ bind(*entry());
1414
1415 Label L_done;
1416
1417 // If we need to load ourselves, do it here.
1418 if (_do_load) {
1419 if (_narrow) {
1420 __ movl(_obj, _addr);
1421 } else {
1422 __ movq(_obj, _addr);
1423 }
1424 }
1425
1426 // If the object is null, there is no point in applying barriers.
1427 if (_narrow) {
1428 __ testl(_obj, _obj);
1429 } else {
1430 __ testptr(_obj, _obj);
1431 }
1432 __ jcc(Assembler::zero, L_done);
1433
1434 // Barriers need temp to work, allocate one now.
1435 bool tmp_live;
1436 Register tmp = select_temp_register(tmp_live, _addr, _obj);
1437 if (tmp_live) {
1438 push_save_register(&masm, tmp);
1439 }
1440
1441 // If object is narrow, we need to decode it first: barrier checks need full oops.
1442 if (_narrow) {
1443 __ decode_heap_oop_not_null(_obj);
1444 }
1445
1446 // Go for barriers. If both barriers are required (rare), do a runtime check for enabled barrier.
1447 if (_needs_keep_alive_barrier) {
1448 keepalive(&masm, _obj, tmp, noreg);
1449 }
1450 if (_needs_load_ref_barrier) {
1451 lrb(&masm, _obj, _addr, tmp);
1452 }
1453
1454 if (tmp_live) {
1455 pop_save_register(&masm, tmp);
1456 }
1457
1458 // If object is narrow, we need to encode it before exiting.
1459 // For encoding, dst can only turn null if we are dealing with weak loads.
1460 // Otherwise, we have already null-checked. We can skip all this if we performed
1461 // the load ourselves, which means the value is not used by caller.
1462 if (_narrow && !_do_load) {
1463 if (_needs_load_ref_weak_barrier) {
1464 __ encode_heap_oop(_obj);
1465 } else {
1466 __ encode_heap_oop_not_null(_obj);
1467 }
1468 }
1469 __ bind(L_done);
1470 __ jmp(*continuation());
1471 }
1472
1473 Label* ShenandoahBarrierStubC2::entry() {
1474 return BarrierStubC2::entry();
1475 }
1476
1477 ShenandoahBarrierStubC2::ShenandoahBarrierStubC2(const MachNode* node, Register obj, Address addr, bool narrow, bool do_load, int offset) : BarrierStubC2(node),
1478 _obj(obj),
1479 _addr(addr),
1480 _do_load(do_load),
1481 _narrow(narrow),
1482 _maybe_null(!src_not_null(node)),
1483 _needs_load_ref_barrier(needs_load_ref_barrier(node)),
1484 _needs_load_ref_weak_barrier(needs_load_ref_barrier_weak(node)),
1485 _needs_keep_alive_barrier(needs_keep_alive_barrier(node)),
1486 _fastpath_branch_offset(),
1487 _test_and_branch_reachable(),
1488 _skip_trampoline(),
1489 _test_and_branch_reachable_entry() {
1490
1491 ShenandoahBarrierStubC2(node, obj, addr, narrow, do_load);
1492 }
1493 #undef __
1494 #endif
|