31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahForwarding.hpp"
35 #include "gc/shenandoah/shenandoahHeap.hpp"
36 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
37 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "macroAssembler_ppc.hpp"
42 #include "runtime/javaThread.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/globalDefinitions.hpp"
45 #include "vm_version_ppc.hpp"
46 #ifdef COMPILER1
47 #include "c1/c1_LIRAssembler.hpp"
48 #include "c1/c1_MacroAssembler.hpp"
49 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
50 #endif
51
52 #define __ masm->
53
54 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
55 Register base, RegisterOrConstant ind_or_offs,
56 Register tmp1, Register tmp2, Register tmp3,
57 MacroAssembler::PreservationLevel preservation_level) {
58 if (ShenandoahSATBBarrier) {
59 __ block_comment("satb_barrier (shenandoahgc) {");
60 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
61 __ block_comment("} satb_barrier (shenandoahgc)");
62 }
63 }
64
65 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
66 Register base, RegisterOrConstant ind_or_offs,
67 Register dst,
68 Register tmp1, Register tmp2,
69 MacroAssembler::PreservationLevel preservation_level) {
70 if (ShenandoahLoadRefBarrier) {
1074 assert(jrt_address != nullptr, "load reference barrier runtime routine cannot be found");
1075
1076 __ save_LR(R11_tmp);
1077 __ push_frame_reg_args(nbytes_save, R11_tmp);
1078
1079 // Invoke runtime. Arguments are already stored in the corresponding registers.
1080 __ call_VM_leaf(jrt_address, R3_obj, R4_load_addr);
1081
1082 // Restore to-be-preserved registers.
1083 __ pop_frame();
1084 __ restore_LR(R11_tmp);
1085 __ restore_volatile_gprs(R1_SP, -nbytes_save, true, false); // Skip 'R3_RET' register.
1086
1087 __ blr();
1088 __ block_comment("} generate_c1_load_reference_barrier_runtime_stub (shenandoahgc)");
1089 }
1090
1091 #undef __
1092
1093 #endif // COMPILER1
|
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahForwarding.hpp"
35 #include "gc/shenandoah/shenandoahHeap.hpp"
36 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
37 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "macroAssembler_ppc.hpp"
42 #include "runtime/javaThread.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/globalDefinitions.hpp"
45 #include "vm_version_ppc.hpp"
46 #ifdef COMPILER1
47 #include "c1/c1_LIRAssembler.hpp"
48 #include "c1/c1_MacroAssembler.hpp"
49 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
50 #endif
51 #ifdef COMPILER2
52 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
53 #endif
54
55 #define __ masm->
56
57 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
58 Register base, RegisterOrConstant ind_or_offs,
59 Register tmp1, Register tmp2, Register tmp3,
60 MacroAssembler::PreservationLevel preservation_level) {
61 if (ShenandoahSATBBarrier) {
62 __ block_comment("satb_barrier (shenandoahgc) {");
63 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
64 __ block_comment("} satb_barrier (shenandoahgc)");
65 }
66 }
67
68 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
69 Register base, RegisterOrConstant ind_or_offs,
70 Register dst,
71 Register tmp1, Register tmp2,
72 MacroAssembler::PreservationLevel preservation_level) {
73 if (ShenandoahLoadRefBarrier) {
1077 assert(jrt_address != nullptr, "load reference barrier runtime routine cannot be found");
1078
1079 __ save_LR(R11_tmp);
1080 __ push_frame_reg_args(nbytes_save, R11_tmp);
1081
1082 // Invoke runtime. Arguments are already stored in the corresponding registers.
1083 __ call_VM_leaf(jrt_address, R3_obj, R4_load_addr);
1084
1085 // Restore to-be-preserved registers.
1086 __ pop_frame();
1087 __ restore_LR(R11_tmp);
1088 __ restore_volatile_gprs(R1_SP, -nbytes_save, true, false); // Skip 'R3_RET' register.
1089
1090 __ blr();
1091 __ block_comment("} generate_c1_load_reference_barrier_runtime_stub (shenandoahgc)");
1092 }
1093
1094 #undef __
1095
1096 #endif // COMPILER1
1097
1098 #ifdef COMPILER2
1099 #undef __
1100 #define __ masm.
1101
1102 int ShenandoahBarrierStubC2::available_gp_registers() {
1103 return 0;
1104 }
1105
1106 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1107 return true;
1108 }
1109
1110 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
1111 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1112
1113 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)), R16_thread);
1114 __ cmpdi(CR0, tmp, 0);
1115 __ beq(CR0, *continuation());
1116 __ b(*entry());
1117
1118 // This is were the slowpath stub will return to or the code above will jump
1119 // to if the checks are false
1120 __ bind(*continuation());
1121 }
1122
1123 #undef __
1124 #define __ masm->
1125
1126 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr, Register oldval,
1127 Register newval, Register tmp1, Register tmp2, Register tmp3, bool exchange, bool narrow, bool weak, bool acquire) {
1128
1129 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp1, addr, tmp2, tmp3, narrow);
1130
1131 Register dest_current = exchange ? res : R0;
1132 Register int_flag = exchange ? noreg : res;
1133 int semantics = MacroAssembler::MemBarNone;
1134
1135 if (narrow) {
1136 semantics = weak && support_IRIW_for_not_multiple_copy_atomic_cpu ? MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter;
1137
1138 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
1139 __ cmpxchgw(CR0, dest_current, oldval, newval, addr,
1140 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
1141 int_flag, nullptr, true, weak);
1142 } else {
1143 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
1144 __ cmpxchgd(CR0, dest_current, oldval, newval, addr,
1145 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
1146 int_flag, nullptr, true, weak);
1147 }
1148
1149 if (exchange) {
1150 if (acquire) {
1151 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
1152 __ isync();
1153 } else {
1154 // isync would be sufficient in case of CompareAndExchangeAcquire, but
1155 // we currently don't optimize for that.
1156 __ sync();
1157 }
1158 }
1159 } else {
1160 if (!weak) {
1161 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
1162 __ isync();
1163 } else {
1164 __ sync();
1165 }
1166 }
1167 }
1168
1169 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
1170 }
1171
1172 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval, Register newval, Register addr, Register tmp1, Register tmp2, Register tmp3) {
1173 bool is_narrow = node->bottom_type()->isa_narrowoop();
1174
1175 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp1, addr, tmp2, tmp3, is_narrow);
1176
1177 if (is_narrow) {
1178 __ getandsetw(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
1179 } else {
1180 __ getandsetd(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
1181 }
1182
1183 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
1184 __ isync();
1185 } else {
1186 __ sync();
1187 }
1188
1189 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
1190 }
1191
1192 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
1193 Register dst, int disp, bool dst_narrow, Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
1194
1195 ShenandoahBarrierStubC2::store_pre(masm, node, tmp1, Address(dst, disp), tmp2, tmp3, dst_narrow);
1196
1197 if (dst_narrow && !src_narrow) {
1198 // Need to encode into tmp, because we cannot clobber src.
1199 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
1200 __ encode_heap_oop(tmp1, src);
1201 } else {
1202 __ encode_heap_oop_not_null(tmp1, src);
1203 }
1204 src = tmp1;
1205 }
1206 if (dst_narrow) {
1207 __ stw(src, disp, dst);
1208 } else {
1209 __ std(src, disp, dst);
1210 }
1211
1212 ShenandoahBarrierStubC2::store_post(masm, node, Address(dst, disp), tmp1, tmp2);
1213 }
1214
1215 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Register addr, int disp, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
1216 if (is_narrow) {
1217 __ lwz(dst, disp, addr);
1218 } else {
1219 __ ld(dst, disp, addr);
1220 }
1221 if (is_acquire) {
1222 __ twi_0(dst);
1223 __ isync();
1224 }
1225
1226 ShenandoahBarrierStubC2::load_post(masm, node, dst, Address(addr, disp), tmp1, tmp2, is_narrow);
1227 }
1228
1229 void ShenandoahBarrierStubC2::store_post(MacroAssembler* masm, const MachNode* node, Address address, Register tmp1, Register tmp2) {
1230 if (!ShenandoahBarrierStubC2::needs_card_barrier(node)) {
1231 return;
1232 }
1233
1234 Assembler::InlineSkippedInstructionsCounter skip_counter(masm);
1235 assert_different_registers(tmp1, tmp2, address.index(), address.base());
1236
1237 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
1238 if (address.index() == noreg) {
1239 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
1240 } else {
1241 __ add(tmp2, address.index(), address.base());
1242 if (address.disp() != 0) {
1243 __ addi(tmp2, tmp2, address.disp());
1244 }
1245 }
1246 __ srdi(tmp2, tmp2, CardTable::card_shift());
1247 __ li(R0, CardTable::dirty_card_val());
1248 __ stbx(R0, tmp2, tmp1);
1249 }
1250
1251 void ShenandoahBarrierStubC2::load_store_post(MacroAssembler* masm, const MachNode* node, Address address, Register tmp1, Register tmp2) {
1252 store_post(masm, node, address, tmp1, tmp2);
1253 }
1254
1255 #undef __
1256 #define __ masm.
1257
1258 void ShenandoahBarrierStubC2::post_init() {
1259 // Do nothing.
1260 }
1261
1262 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1263 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1264 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1265
1266 __ bind(*entry());
1267
1268 // If we need to load ourselves, do it here.
1269 if (_do_load) {
1270 if (_narrow) {
1271 __ lwz(_obj, _addr.disp(), _addr.base());
1272 } else {
1273 __ ld(_obj, _addr.disp(), _addr.base());
1274 }
1275 }
1276
1277 // If the object is null, there is no point in applying barriers.
1278 maybe_far_jump_if_zero(masm, _obj, continuation());
1279
1280 // We need to make sure that loads done by callers survive across slow-path calls.
1281 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1282 if (!_do_load || (_needs_keep_alive_barrier && _needs_load_ref_barrier)) {
1283 preserve(_obj);
1284 }
1285
1286 // Go for barriers. Barriers can return straight to continuation, as long
1287 // as another barrier is not needed and we can reach the fastpath.
1288 if (_needs_keep_alive_barrier && _needs_load_ref_barrier) {
1289 keepalive(masm, nullptr);
1290 lrb(masm);
1291 } else if (_needs_keep_alive_barrier) {
1292 keepalive(masm, continuation());
1293 } else if (_needs_load_ref_barrier) {
1294 lrb(masm);
1295 } else {
1296 ShouldNotReachHere();
1297 }
1298 }
1299
1300 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg, Label* L_done) {
1301 Label L_short_jump;
1302 __ cmpdi(CR0, reg, 0);
1303 __ bne(CR0, L_short_jump);
1304 __ b(*L_done);
1305 __ bind(L_short_jump);
1306 }
1307
1308 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1309 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1310 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1311
1312 Label L_through, L_slowpath;
1313
1314 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1315 if (_needs_load_ref_barrier) {
1316 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
1317 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING)), R16_thread);
1318 __ cmpdi(CR0, _tmp1, 0);
1319 __ beq(CR0, L_through);
1320 }
1321
1322 // Fast-path: put object into buffer.
1323 // If buffer is already full, go slow.
1324 __ ld(_tmp1, index_offset, R16_thread);
1325 __ cmpdi(CR0, _tmp1, 0);
1326 __ beq(CR0, L_slowpath);
1327 __ addi(_tmp1, _tmp1, -wordSize);
1328 __ std(_tmp1, index_offset, R16_thread);
1329 __ ld(_tmp2, buffer_offset, R16_thread);
1330
1331 // If object is narrow, we need to unpack it before inserting into buffer,
1332 // and pack it back. We can skip the unpack if we know that object is not preserved.
1333 if (_narrow) {
1334 __ decode_heap_oop_not_null(_obj);
1335 }
1336 __ stdx(_obj, _tmp2, _tmp1);
1337 if (_narrow && is_preserved(_obj)) {
1338 __ encode_heap_oop_not_null(_obj);
1339 }
1340
1341 // Fast-path exits here.
1342 if (L_done != nullptr) {
1343 __ b(*L_done);
1344 } else {
1345 __ b(L_through);
1346 }
1347
1348 // Slow-path: call runtime to handle.
1349 __ bind(L_slowpath);
1350 {
1351 SaveLiveRegisters slr(&masm, this);
1352
1353 // Go to runtime and handle the rest there.
1354 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1355 }
1356
1357 if (L_done != nullptr) {
1358 __ b(*L_done);
1359 } else {
1360 __ bind(L_through);
1361 }
1362 }
1363
1364 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1365 Label L_slow;
1366
1367 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1368 if (_needs_keep_alive_barrier) {
1369 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1370 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)), R16_thread);
1371 maybe_far_jump_if_zero(masm, _tmp1, continuation());
1372 }
1373
1374 // If weak references are being processed, weak/phantom loads need to go slow,
1375 // regardless of their cset status.
1376 if (_needs_load_ref_weak_barrier) {
1377 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)), R16_thread);
1378 __ cmpdi(CR0, _tmp1, 0);
1379 __ bne(CR0, L_slow);
1380 }
1381
1382 // Cset-check. Fall-through to slow if in collection set.
1383 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1384 if (_narrow) {
1385 __ decode_heap_oop_not_null(_tmp2, _obj);
1386 } else {
1387 __ mr(_tmp2, _obj);
1388 }
1389 __ srdi(_tmp2, _tmp2, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1390 __ lbzx(_tmp2, _tmp2, _tmp1);
1391 maybe_far_jump_if_zero(masm, _tmp2, continuation());
1392
1393 // Slow path
1394 __ bind(L_slow);
1395
1396 // Obj is the result, need to temporarily stop preserving it.
1397 bool is_obj_preserved = is_preserved(_obj);
1398 if (is_obj_preserved) {
1399 dont_preserve(_obj);
1400 }
1401 {
1402 SaveLiveRegisters slr(&masm, this);
1403
1404 // Shuffle in the arguments. The end result should be:
1405 // c_rarg0 <-- obj
1406 // c_rarg1 <-- lea(addr)
1407 Register c_rarg0 = R3_ARG1;
1408 Register c_rarg1 = R4_ARG2;
1409 if (c_rarg0 == _obj) {
1410 __ addi(c_rarg1, _addr.base(), _addr.disp());
1411 } else if (c_rarg1 == _obj) {
1412 // Set up arguments in reverse, and then flip them
1413 __ addi(c_rarg0, _addr.base(), _addr.disp());
1414 // flip them
1415 __ mr(_tmp1, c_rarg0);
1416 __ mr(c_rarg0, c_rarg1);
1417 __ mr(c_rarg1, _tmp1);
1418 } else {
1419 assert_different_registers(c_rarg1, _obj);
1420 __ addi(c_rarg1, _addr.base(), _addr.disp());
1421 __ mr(c_rarg0, _obj);
1422 }
1423
1424 // Go to runtime and handle the rest there.
1425 __ call_VM_leaf(lrb_runtime_entry_addr(), c_rarg0, c_rarg1);
1426
1427 // Save the result where needed.
1428 if (_obj != R3_RET) {
1429 __ mr(_obj, R3_RET);
1430 }
1431 }
1432 if (is_obj_preserved) {
1433 preserve(_obj);
1434 }
1435
1436 __ b(*continuation());
1437 }
1438
1439 #undef __
1440 #define __ masm->
1441
1442 #endif // COMPILER2
|