31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahForwarding.hpp"
35 #include "gc/shenandoah/shenandoahHeap.hpp"
36 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
37 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "macroAssembler_ppc.hpp"
42 #include "runtime/javaThread.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/globalDefinitions.hpp"
45 #include "vm_version_ppc.hpp"
46 #ifdef COMPILER1
47 #include "c1/c1_LIRAssembler.hpp"
48 #include "c1/c1_MacroAssembler.hpp"
49 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
50 #endif
51
52 #define __ masm->
53
54 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
55 Register base, RegisterOrConstant ind_or_offs,
56 Register tmp1, Register tmp2, Register tmp3,
57 MacroAssembler::PreservationLevel preservation_level) {
58 if (ShenandoahSATBBarrier) {
59 __ block_comment("satb_barrier (shenandoahgc) {");
60 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
61 __ block_comment("} satb_barrier (shenandoahgc)");
62 }
63 }
64
65 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
66 Register base, RegisterOrConstant ind_or_offs,
67 Register dst,
68 Register tmp1, Register tmp2,
69 MacroAssembler::PreservationLevel preservation_level) {
70 if (ShenandoahLoadRefBarrier) {
1072 assert(jrt_address != nullptr, "load reference barrier runtime routine cannot be found");
1073
1074 __ save_LR(R11_tmp);
1075 __ push_frame_reg_args(nbytes_save, R11_tmp);
1076
1077 // Invoke runtime. Arguments are already stored in the corresponding registers.
1078 __ call_VM_leaf(jrt_address, R3_obj, R4_load_addr);
1079
1080 // Restore to-be-preserved registers.
1081 __ pop_frame();
1082 __ restore_LR(R11_tmp);
1083 __ restore_volatile_gprs(R1_SP, -nbytes_save, true, false); // Skip 'R3_RET' register.
1084
1085 __ blr();
1086 __ block_comment("} generate_c1_load_reference_barrier_runtime_stub (shenandoahgc)");
1087 }
1088
1089 #undef __
1090
1091 #endif // COMPILER1
|
31 #include "gc/shenandoah/mode/shenandoahMode.hpp"
32 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
33 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
34 #include "gc/shenandoah/shenandoahForwarding.hpp"
35 #include "gc/shenandoah/shenandoahHeap.hpp"
36 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
37 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
38 #include "gc/shenandoah/shenandoahRuntime.hpp"
39 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
40 #include "interpreter/interpreter.hpp"
41 #include "macroAssembler_ppc.hpp"
42 #include "runtime/javaThread.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "utilities/globalDefinitions.hpp"
45 #include "vm_version_ppc.hpp"
46 #ifdef COMPILER1
47 #include "c1/c1_LIRAssembler.hpp"
48 #include "c1/c1_MacroAssembler.hpp"
49 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
50 #endif
51 #ifdef COMPILER2
52 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
53 #endif
54
55 #define __ masm->
56
57 void ShenandoahBarrierSetAssembler::satb_barrier(MacroAssembler *masm,
58 Register base, RegisterOrConstant ind_or_offs,
59 Register tmp1, Register tmp2, Register tmp3,
60 MacroAssembler::PreservationLevel preservation_level) {
61 if (ShenandoahSATBBarrier) {
62 __ block_comment("satb_barrier (shenandoahgc) {");
63 satb_barrier_impl(masm, 0, base, ind_or_offs, tmp1, tmp2, tmp3, preservation_level);
64 __ block_comment("} satb_barrier (shenandoahgc)");
65 }
66 }
67
68 void ShenandoahBarrierSetAssembler::load_reference_barrier(MacroAssembler *masm, DecoratorSet decorators,
69 Register base, RegisterOrConstant ind_or_offs,
70 Register dst,
71 Register tmp1, Register tmp2,
72 MacroAssembler::PreservationLevel preservation_level) {
73 if (ShenandoahLoadRefBarrier) {
1075 assert(jrt_address != nullptr, "load reference barrier runtime routine cannot be found");
1076
1077 __ save_LR(R11_tmp);
1078 __ push_frame_reg_args(nbytes_save, R11_tmp);
1079
1080 // Invoke runtime. Arguments are already stored in the corresponding registers.
1081 __ call_VM_leaf(jrt_address, R3_obj, R4_load_addr);
1082
1083 // Restore to-be-preserved registers.
1084 __ pop_frame();
1085 __ restore_LR(R11_tmp);
1086 __ restore_volatile_gprs(R1_SP, -nbytes_save, true, false); // Skip 'R3_RET' register.
1087
1088 __ blr();
1089 __ block_comment("} generate_c1_load_reference_barrier_runtime_stub (shenandoahgc)");
1090 }
1091
1092 #undef __
1093
1094 #endif // COMPILER1
1095
1096 #ifdef COMPILER2
1097
1098 #undef __
1099 #define __ masm->
1100
1101 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Register addr, int disp, Register tmp1, Register tmp2, bool is_narrow, bool is_acquire) {
1102 if (is_narrow) {
1103 __ lwz(dst, disp, addr);
1104 } else {
1105 __ ld(dst, disp, addr);
1106 }
1107 if (is_acquire) {
1108 __ twi_0(dst);
1109 __ isync();
1110 }
1111
1112 ShenandoahBarrierStubC2::load_post(masm, node, dst, Address(addr, disp), tmp1, tmp2, is_narrow);
1113 }
1114
1115 void ShenandoahBarrierSetAssembler::store_c2(const MachNode* node, MacroAssembler* masm,
1116 Register dst, int disp, bool dst_narrow, Register src, bool src_narrow, Register tmp1, Register tmp2, Register tmp3) {
1117
1118 ShenandoahBarrierStubC2::store_pre(masm, node, tmp1, Address(dst, disp), tmp2, tmp3, dst_narrow);
1119
1120 if (dst_narrow && !src_narrow) {
1121 // Need to encode into tmp, because we cannot clobber src.
1122 if ((node->barrier_data() & ShenandoahBitNotNull) == 0) {
1123 src = __ encode_heap_oop(tmp1, src);
1124 } else {
1125 src = __ encode_heap_oop_not_null(tmp1, src);
1126 }
1127 }
1128 if (dst_narrow) {
1129 __ stw(src, disp, dst);
1130 } else {
1131 __ std(src, disp, dst);
1132 }
1133
1134 ShenandoahBarrierStubC2::store_post(masm, node, Address(dst, disp), tmp1, tmp2);
1135 }
1136
1137 void ShenandoahBarrierSetAssembler::compare_and_set_c2(const MachNode* node, MacroAssembler* masm, Register res, Register addr, Register oldval,
1138 Register newval, Register tmp1, Register tmp2, Register tmp3, bool exchange, bool narrow, bool weak, bool acquire) {
1139
1140 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp1, addr, tmp2, tmp3, narrow);
1141
1142 Register dest_current = exchange ? res : R0;
1143 Register int_flag = exchange ? noreg : res;
1144 int semantics = MacroAssembler::MemBarNone;
1145
1146 if (acquire) {
1147 semantics = support_IRIW_for_not_multiple_copy_atomic_cpu ?
1148 MacroAssembler::MemBarAcq : MacroAssembler::MemBarFenceAfter;
1149 }
1150
1151 if (narrow) {
1152 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
1153 __ cmpxchgw(CR0, dest_current, oldval, newval, addr,
1154 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
1155 int_flag, nullptr, true, weak);
1156 } else {
1157 // CmpxchgX sets CR0 to cmpX(src1, src2) and Rres to 'true'/'false'.
1158 __ cmpxchgd(CR0, dest_current, oldval, newval, addr,
1159 semantics, MacroAssembler::cmpxchgx_hint_atomic_update(),
1160 int_flag, nullptr, true, weak);
1161 }
1162
1163 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
1164 }
1165
1166 void ShenandoahBarrierSetAssembler::get_and_set_c2(const MachNode* node, MacroAssembler* masm, Register preval, Register newval, Register addr, Register tmp1, Register tmp2, Register tmp3) {
1167 bool is_narrow = node->bottom_type()->isa_narrowoop();
1168
1169 ShenandoahBarrierStubC2::load_store_pre(masm, node, tmp1, addr, tmp2, tmp3, is_narrow);
1170
1171 if (is_narrow) {
1172 __ getandsetw(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
1173 } else {
1174 __ getandsetd(preval, newval, addr, MacroAssembler::cmpxchgx_hint_atomic_update());
1175 }
1176
1177 if (support_IRIW_for_not_multiple_copy_atomic_cpu) {
1178 __ isync();
1179 } else {
1180 __ sync();
1181 }
1182
1183 ShenandoahBarrierStubC2::load_store_post(masm, node, Address(addr, 0), tmp1, tmp2);
1184 }
1185
1186 #undef __
1187 #define __ masm.
1188
1189 void ShenandoahBarrierStubC2::cardtable(MacroAssembler& masm, Address address, Register tmp1, Register tmp2) {
1190 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1191 assert_different_registers(tmp1, tmp2, address.index(), address.base());
1192
1193 __ ld(tmp1, in_bytes(ShenandoahThreadLocalData::card_table_offset()), R16_thread);
1194 if (address.index() == noreg) {
1195 __ add_const_optimized(tmp2, address.base(), address.disp(), R0);
1196 } else {
1197 __ add(tmp2, address.index(), address.base());
1198 if (address.disp() != 0) {
1199 __ addi(tmp2, tmp2, address.disp());
1200 }
1201 }
1202 __ srdi(tmp2, tmp2, CardTable::card_shift());
1203 __ li(R0, CardTable::dirty_card_val());
1204 __ stbx(R0, tmp2, tmp1);
1205 }
1206
1207 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state, Register tmp) {
1208 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1209
1210 __ lbz(tmp, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(test_state)), R16_thread);
1211 __ cmpdi(CR0, tmp, 0);
1212 // Branch to entry if not equal
1213 __ bc_far_optimized(Assembler::bcondCRbiIs0, __ bi0(CR0, Assembler::equal), *entry());
1214 // This is were the slowpath stub will return to
1215 __ bind(*continuation());
1216 }
1217
1218 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1219 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1220 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1221
1222 __ bind(*entry());
1223
1224 // If we need to load ourselves, do it here.
1225 if (_do_load) {
1226 if (_narrow) {
1227 __ lwz(_obj, _addr.disp(), _addr.base());
1228 } else {
1229 __ ld(_obj, _addr.disp(), _addr.base());
1230 }
1231 }
1232
1233 // If the object is null, there is no point in applying barriers.
1234 maybe_far_jump_if_zero(masm, _obj);
1235
1236 // We need to make sure that loads done by callers survive across slow-path calls.
1237 // For self-loads, we need to care about the case when both KA and LRB are enabled (rare).
1238 bool needs_both_barriers = _needs_keep_alive_barrier && _needs_load_ref_barrier;
1239 if (!_do_load || needs_both_barriers) {
1240 preserve(_obj);
1241 }
1242
1243 // Go for barriers. Barriers can return straight to continuation, as long
1244 // as another barrier is not needed and we can reach the fastpath.
1245 if (needs_both_barriers) {
1246 keepalive(masm, nullptr);
1247 lrb(masm);
1248 } else if (_needs_keep_alive_barrier) {
1249 keepalive(masm, continuation());
1250 } else if (_needs_load_ref_barrier) {
1251 lrb(masm);
1252 } else {
1253 ShouldNotReachHere();
1254 }
1255 }
1256
1257 void ShenandoahBarrierStubC2::maybe_far_jump_if_zero(MacroAssembler& masm, Register reg) {
1258 __ cmpdi(CR0, reg, 0);
1259 // Branch to continuation if equal
1260 __ bc_far_optimized(Assembler::bcondCRbiIs1, __ bi0(CR0, Assembler::equal), *continuation());
1261 }
1262
1263 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Label* L_done) {
1264 const int gcstate_offset = in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::MARKING));
1265 const int index_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset());
1266 const int buffer_offset = in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset());
1267 Label L_through, L_slowpath;
1268
1269 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1270 if (_needs_load_ref_barrier) {
1271 assert(L_done == nullptr, "L_done is always null when _needs_load_ref_barrier is true");
1272 __ lbz(_tmp1, gcstate_offset, R16_thread);
1273 __ cmpdi(CR0, _tmp1, 0);
1274 __ beq(CR0, L_through);
1275 }
1276
1277 // Fast-path: put object into buffer.
1278 // If buffer is already full, go slow.
1279 __ ld(_tmp1, index_offset, R16_thread);
1280 __ cmpdi(CR0, _tmp1, 0);
1281 __ beq(CR0, L_slowpath);
1282 __ addi(_tmp1, _tmp1, -wordSize);
1283 __ std(_tmp1, index_offset, R16_thread);
1284 __ ld(_tmp2, buffer_offset, R16_thread);
1285
1286 // Store the object in queue.
1287 // If object is narrow, we need to decode it before inserting.
1288 if (_narrow) {
1289 __ add(_tmp2, _tmp2, _tmp1);
1290 Register decoded = __ decode_heap_oop_not_null(_tmp1, _obj);
1291 __ stdx(decoded, _tmp2);
1292 } else {
1293 __ stdx(_obj, _tmp2, _tmp1);
1294 }
1295
1296 // Fast-path exits here.
1297 if (L_done != nullptr) {
1298 __ b(*L_done);
1299 } else {
1300 __ b(L_through);
1301 }
1302
1303 // Slow-path: call runtime to handle.
1304 __ bind(L_slowpath);
1305
1306 {
1307 SaveLiveRegisters slr(&masm, this);
1308
1309 // Go to runtime and handle the rest there.
1310 __ call_VM_leaf(keepalive_runtime_entry_addr(), _obj);
1311 }
1312
1313 if (L_done != nullptr) {
1314 __ b(*L_done);
1315 } else {
1316 __ bind(L_through);
1317 }
1318 }
1319
1320 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm) {
1321 Label L_slow;
1322
1323 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1324 if (_needs_keep_alive_barrier) {
1325 char state_to_check = ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0);
1326 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(state_to_check)), R16_thread);
1327 maybe_far_jump_if_zero(masm, _tmp1);
1328 }
1329
1330 // If weak references are being processed, weak/phantom loads need to go slow,
1331 // regardless of their cset status.
1332 if (_needs_load_ref_weak_barrier) {
1333 __ lbz(_tmp1, in_bytes(ShenandoahThreadLocalData::gc_state_fast_array_offset(ShenandoahHeap::WEAK_ROOTS)), R16_thread);
1334 __ cmpdi(CR0, _tmp1, 0);
1335 __ bne(CR0, L_slow);
1336 }
1337
1338 // Cset-check. Fall-through to slow if in collection set.
1339 __ load_const_optimized(_tmp1, ShenandoahHeap::in_cset_fast_test_addr(), _tmp2);
1340 if (_narrow) {
1341 Register decoded = __ decode_heap_oop_not_null(_tmp2, _obj);
1342 __ srdi(_tmp2, decoded, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1343 } else {
1344 __ srdi(_tmp2, _obj, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1345 }
1346 __ lbzx(_tmp2, _tmp2, _tmp1);
1347 maybe_far_jump_if_zero(masm, _tmp2);
1348
1349 // Slow path
1350 __ bind(L_slow);
1351
1352 // Obj is the result, need to temporarily stop preserving it.
1353 bool is_obj_preserved = is_preserved(_obj);
1354 if (is_obj_preserved) {
1355 dont_preserve(_obj);
1356 }
1357 {
1358 SaveLiveRegisters slr(&masm, this);
1359
1360 // Shuffle in the arguments. The end result should be:
1361 // c_rarg0 <-- obj
1362 // c_rarg1 <-- lea(addr)
1363 Register c_rarg0 = R3_ARG1;
1364 Register c_rarg1 = R4_ARG2;
1365 if (c_rarg0 == _obj) {
1366 __ addi(c_rarg1, _addr.base(), _addr.disp());
1367 } else if (c_rarg1 == _obj) {
1368 // Set up arguments in reverse, and then flip them
1369 __ addi(c_rarg0, _addr.base(), _addr.disp());
1370 // flip them
1371 __ mr(_tmp1, c_rarg0);
1372 __ mr(c_rarg0, c_rarg1);
1373 __ mr(c_rarg1, _tmp1);
1374 } else {
1375 assert_different_registers(c_rarg1, _obj);
1376 __ addi(c_rarg1, _addr.base(), _addr.disp());
1377 __ mr(c_rarg0, _obj);
1378 }
1379
1380 // Go to runtime and handle the rest there.
1381 __ call_VM_leaf(lrb_runtime_entry_addr(), c_rarg0, c_rarg1);
1382
1383 // Save the result where needed.
1384 if (_obj != R3_RET) {
1385 __ mr(_obj, R3_RET);
1386 }
1387 }
1388 if (is_obj_preserved) {
1389 preserve(_obj);
1390 }
1391
1392 __ b(*continuation());
1393 }
1394
1395 int ShenandoahBarrierStubC2::available_gp_registers() {
1396 Unimplemented(); // Not used
1397 return 0;
1398 }
1399
1400 bool ShenandoahBarrierStubC2::is_special_register(Register r) {
1401 Unimplemented(); // Not used
1402 return true;
1403 }
1404
1405 void ShenandoahBarrierStubC2::post_init() {
1406 // Do nothing.
1407 }
1408
1409 #endif // COMPILER2
|