17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #endif
48
49 #define __ masm->
50
51 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
52 if (handle_gpr) {
53 __ push_IU_state();
54 }
55
56 if (handle_fp) {
1169 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
1170 Label L_done;
1171 if (UseCondCardMark) {
1172 __ cmpb(card_address, 0);
1173 __ jccb(Assembler::equal, L_done);
1174 }
1175 if (UseCompressedOops && CompressedOops::base() == nullptr) {
1176 __ movb(card_address, r12);
1177 } else {
1178 __ movb(card_address, 0);
1179 }
1180 __ bind(L_done);
1181 }
1182
1183 #undef __
1184 #define __ masm.
1185
1186 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
1187 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1188
1189 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
1190 __ testb(gc_state_fast, ShenandoahThreadLocalData::gc_state_to_fast(test_state));
1191 __ jcc(Assembler::notZero, *entry());
1192 __ bind(*continuation());
1193 }
1194
1195 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1196 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1197
1198 __ bind(*entry());
1199
1200 Label L_done, L_barriers;
1201
1202 // If we need to load ourselves, do it here.
1203 if (_do_load) {
1204 if (_narrow) {
1205 __ movl(_obj, _addr);
1206 } else {
1207 __ movq(_obj, _addr);
1208 }
1209 }
1210
1211 // If the object is null, there is no point in applying barriers.
1212 if (_narrow) {
1213 __ testl(_obj, _obj);
1214 } else {
1250 // If object is narrow, we need to encode it before exiting.
1251 // For encoding, dst can only turn null if we are dealing with weak loads.
1252 // Otherwise, we have already null-checked. We can skip all this if we performed
1253 // the load ourselves, which means the value is not used by caller.
1254 if (_narrow && !_do_load) {
1255 if (_needs_load_ref_weak_barrier) {
1256 __ encode_heap_oop(_obj);
1257 } else {
1258 __ encode_heap_oop_not_null(_obj);
1259 }
1260 }
1261 __ jmp(L_done);
1262 }
1263
1264 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Register obj, Register tmp1) {
1265 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1266 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1267
1268 Label L_fast, L_done;
1269
1270 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1271 if (_needs_load_ref_barrier) {
1272 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1273 __ testb(gc_state, ShenandoahHeap::MARKING);
1274 __ jccb(Assembler::zero, L_done);
1275 }
1276
1277 // Check if buffer is already full. Go slow, if so.
1278 __ movptr(tmp1, index);
1279 __ testptr(tmp1, tmp1);
1280 __ jccb(Assembler::notZero, L_fast);
1281
1282 // Slow-path: call runtime to handle.
1283 {
1284 // Shuffle in the arguments. The end result should be:
1285 // c_rarg0 <-- obj
1286 //
1287 // Save clobbered registers before overwriting them.
1288 bool clobbered_c_rarg0 = false;
1289 if (c_rarg0 != obj) {
1290 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1291 __ mov(c_rarg0, obj);
1292 }
1293
1294 // Go to runtime stub and handle the rest there.
1295 __ call(RuntimeAddress(keepalive_runtime_entry_addr()));
1297 // Restore the clobbered registers.
1298 if (clobbered_c_rarg0) {
1299 pop_save_register(masm, c_rarg0);
1300 }
1301 __ jmpb(L_done);
1302 }
1303
1304 // Fast-path: put object into buffer.
1305 __ bind(L_fast);
1306 __ subptr(tmp1, wordSize);
1307 __ movptr(index, tmp1);
1308 __ addptr(tmp1, buffer);
1309 __ movptr(Address(tmp1, 0), obj);
1310
1311 __ bind(L_done);
1312 }
1313
1314 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm, Register obj, Address addr, Register tmp) {
1315 Label L_done, L_slow;
1316
1317 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1318 if (_needs_keep_alive_barrier) {
1319 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1320 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0));
1321 __ jccb(Assembler::zero, L_done);
1322 }
1323
1324 // If weak references are being processed, weak/phantom loads need to go slow,
1325 // regadless of their cset status.
1326 if (_needs_load_ref_weak_barrier) {
1327 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1328 __ testb(gc_state, ShenandoahHeap::WEAK_ROOTS);
1329 __ jccb(Assembler::notZero, L_slow);
1330 }
1331
1332 // Compute the cset bitmap index
1333 __ movptr(tmp, obj);
1334 __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1335
1336 // If cset address is in good spot to just use it as offset. It almost always is.
1337 Address cset_addr_arg;
1338 intptr_t cset_addr = (intptr_t) ShenandoahHeap::in_cset_fast_test_addr();
1339 if ((cset_addr >> 3) < INT32_MAX) {
1340 assert(is_aligned(cset_addr, 8), "Sanity");
1341 cset_addr_arg = Address(tmp, checked_cast<int>(cset_addr >> 3), Address::times_8);
1342 } else {
1477 fallback_live = r;
1478 }
1479 }
1480 }
1481
1482 // If we could not find a non-live register, select the live fallback:
1483 if (tmp == noreg) {
1484 tmp = fallback_live;
1485 selected_live = true;
1486 } else {
1487 selected_live = false;
1488 }
1489
1490 assert(tmp != noreg, "successfully selected");
1491 assert_different_registers(tmp, reg1);
1492 assert_different_registers(tmp, addr.base());
1493 assert_different_registers(tmp, addr.index());
1494 return tmp;
1495 }
1496
1497 void ShenandoahBarrierStubC2::post_init(int offset) {
1498 // Do nothing.
1499 }
1500 #undef __
1501 #endif
|
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "nativeInst_x86.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "utilities/macros.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #endif
49
50 #define __ masm->
51
52 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
53 if (handle_gpr) {
54 __ push_IU_state();
55 }
56
57 if (handle_fp) {
1170 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
1171 Label L_done;
1172 if (UseCondCardMark) {
1173 __ cmpb(card_address, 0);
1174 __ jccb(Assembler::equal, L_done);
1175 }
1176 if (UseCompressedOops && CompressedOops::base() == nullptr) {
1177 __ movb(card_address, r12);
1178 } else {
1179 __ movb(card_address, 0);
1180 }
1181 __ bind(L_done);
1182 }
1183
1184 #undef __
1185 #define __ masm.
1186
1187 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
1188 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1189
1190 // Emit the unconditional branch in the first version of the method.
1191 // Let the rest of runtime figure out how to manage it.
1192 __ relocate(barrier_Relocation::spec());
1193 __ jmp(*entry(), /* maybe_short = */ false);
1194
1195 #ifdef ASSERT
1196 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1197 __ testb(gc_state, 0xFF);
1198 __ jccb(Assembler::zero, *continuation());
1199 __ hlt(); // Correctness bug: barrier is NOP-ed, but heap is NOT IDLE
1200 #endif
1201 // TODO: When barriers are consistently turned off at the end of the cycle, assert that barrier is NOP-ed.
1202
1203 __ bind(*continuation());
1204 }
1205
1206 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
1207 NativeInstruction* ni = nativeInstruction_at(pc);
1208 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
1209 NativeJump* jmp = nativeJump_at(pc);
1210 return jmp->jump_destination();
1211 }
1212
1213 void insert_5_byte_nop(address pc) {
1214 *(pc + 0) = 0x0F;
1215 *(pc + 1) = 0x1F;
1216 *(pc + 2) = 0x44;
1217 *(pc + 3) = 0x00;
1218 *(pc + 4) = 0x00;
1219 ICache::invalidate_range(pc, 5);
1220 }
1221
1222 bool is_5_byte_nop(address pc) {
1223 if (*(pc + 0) != 0x0F) return false;
1224 if (*(pc + 1) != 0x1F) return false;
1225 if (*(pc + 2) != 0x44) return false;
1226 if (*(pc + 3) != 0x00) return false;
1227 if (*(pc + 4) != 0x00) return false;
1228 return true;
1229 }
1230
1231 void check_at(bool cond, address pc, const char* msg) {
1232 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x%02x",
1233 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3), *(pc + 4));
1234 }
1235
1236 void ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
1237 NativeInstruction* ni = nativeInstruction_at(pc);
1238 if (ni->is_jump()) {
1239 insert_5_byte_nop(pc);
1240 } else {
1241 check_at(is_5_byte_nop(pc), pc, "Should already be nop");
1242 }
1243 }
1244
1245 void ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
1246 NativeInstruction* ni = nativeInstruction_at(pc);
1247 if (is_5_byte_nop(pc)) {
1248 NativeJump::insert(pc, stub_addr);
1249 } else {
1250 check_at(ni->is_jump(), pc, "Should already be jump");
1251 check_at(nativeJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
1252 }
1253 }
1254
1255 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1256 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1257
1258 __ bind(*entry());
1259
1260 Label L_done, L_barriers;
1261
1262 // If we need to load ourselves, do it here.
1263 if (_do_load) {
1264 if (_narrow) {
1265 __ movl(_obj, _addr);
1266 } else {
1267 __ movq(_obj, _addr);
1268 }
1269 }
1270
1271 // If the object is null, there is no point in applying barriers.
1272 if (_narrow) {
1273 __ testl(_obj, _obj);
1274 } else {
1310 // If object is narrow, we need to encode it before exiting.
1311 // For encoding, dst can only turn null if we are dealing with weak loads.
1312 // Otherwise, we have already null-checked. We can skip all this if we performed
1313 // the load ourselves, which means the value is not used by caller.
1314 if (_narrow && !_do_load) {
1315 if (_needs_load_ref_weak_barrier) {
1316 __ encode_heap_oop(_obj);
1317 } else {
1318 __ encode_heap_oop_not_null(_obj);
1319 }
1320 }
1321 __ jmp(L_done);
1322 }
1323
1324 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Register obj, Register tmp1) {
1325 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1326 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1327
1328 Label L_fast, L_done;
1329
1330 // Hotpatched GC checks only care about idle/non-idle state, so we need to check again here.
1331 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1332 __ testb(gc_state, ShenandoahHeap::MARKING);
1333 __ jccb(Assembler::zero, L_done);
1334
1335 // Check if buffer is already full. Go slow, if so.
1336 __ movptr(tmp1, index);
1337 __ testptr(tmp1, tmp1);
1338 __ jccb(Assembler::notZero, L_fast);
1339
1340 // Slow-path: call runtime to handle.
1341 {
1342 // Shuffle in the arguments. The end result should be:
1343 // c_rarg0 <-- obj
1344 //
1345 // Save clobbered registers before overwriting them.
1346 bool clobbered_c_rarg0 = false;
1347 if (c_rarg0 != obj) {
1348 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1349 __ mov(c_rarg0, obj);
1350 }
1351
1352 // Go to runtime stub and handle the rest there.
1353 __ call(RuntimeAddress(keepalive_runtime_entry_addr()));
1355 // Restore the clobbered registers.
1356 if (clobbered_c_rarg0) {
1357 pop_save_register(masm, c_rarg0);
1358 }
1359 __ jmpb(L_done);
1360 }
1361
1362 // Fast-path: put object into buffer.
1363 __ bind(L_fast);
1364 __ subptr(tmp1, wordSize);
1365 __ movptr(index, tmp1);
1366 __ addptr(tmp1, buffer);
1367 __ movptr(Address(tmp1, 0), obj);
1368
1369 __ bind(L_done);
1370 }
1371
1372 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm, Register obj, Address addr, Register tmp) {
1373 Label L_done, L_slow;
1374
1375 // Hotpatched GC checks only care about idle/non-idle state, so we need a check here.
1376 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1377 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0));
1378 __ jccb(Assembler::zero, L_done);
1379
1380 // If weak references are being processed, weak/phantom loads need to go slow,
1381 // regadless of their cset status.
1382 if (_needs_load_ref_weak_barrier) {
1383 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1384 __ testb(gc_state, ShenandoahHeap::WEAK_ROOTS);
1385 __ jccb(Assembler::notZero, L_slow);
1386 }
1387
1388 // Compute the cset bitmap index
1389 __ movptr(tmp, obj);
1390 __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1391
1392 // If cset address is in good spot to just use it as offset. It almost always is.
1393 Address cset_addr_arg;
1394 intptr_t cset_addr = (intptr_t) ShenandoahHeap::in_cset_fast_test_addr();
1395 if ((cset_addr >> 3) < INT32_MAX) {
1396 assert(is_aligned(cset_addr, 8), "Sanity");
1397 cset_addr_arg = Address(tmp, checked_cast<int>(cset_addr >> 3), Address::times_8);
1398 } else {
1533 fallback_live = r;
1534 }
1535 }
1536 }
1537
1538 // If we could not find a non-live register, select the live fallback:
1539 if (tmp == noreg) {
1540 tmp = fallback_live;
1541 selected_live = true;
1542 } else {
1543 selected_live = false;
1544 }
1545
1546 assert(tmp != noreg, "successfully selected");
1547 assert_different_registers(tmp, reg1);
1548 assert_different_registers(tmp, addr.base());
1549 assert_different_registers(tmp, addr.index());
1550 return tmp;
1551 }
1552
1553 #undef __
1554 #endif
|