17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "runtime/javaThread.hpp"
38 #include "runtime/sharedRuntime.hpp"
39 #include "utilities/macros.hpp"
40 #ifdef COMPILER1
41 #include "c1/c1_LIRAssembler.hpp"
42 #include "c1/c1_MacroAssembler.hpp"
43 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
44 #endif
45 #ifdef COMPILER2
46 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
47 #endif
48
49 #define __ masm->
50
51 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
52 if (handle_gpr) {
53 __ push_IU_state();
54 }
55
56 if (handle_fp) {
1002 } else {
1003 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
1004 }
1005 } else {
1006 assert(is_phantom, "only remaining strength");
1007 assert(is_native, "phantom must only be called off-heap");
1008 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
1009 }
1010
1011 __ restore_live_registers_except_rax(true);
1012
1013 __ epilogue();
1014 }
1015
1016 #undef __
1017
1018 #endif // COMPILER1
1019
1020 #ifdef COMPILER2
1021
1022 #undef __
1023 #define __ masm->
1024
1025 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, bool narrow) {
1026 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
1027 if (narrow) {
1028 __ movl(dst, src);
1029 } else {
1030 __ movq(dst, src);
1031 }
1032
1033 // Post-barrier: LRB
1034 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
1035 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, dst, src, narrow, false);
1036 char check = 0;
1037 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
1038 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
1039 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
1040 stub->enter_if_gc_state(*masm, check);
1041 }
1169 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
1170 Label L_done;
1171 if (UseCondCardMark) {
1172 __ cmpb(card_address, 0);
1173 __ jccb(Assembler::equal, L_done);
1174 }
1175 if (UseCompressedOops && CompressedOops::base() == nullptr) {
1176 __ movb(card_address, r12);
1177 } else {
1178 __ movb(card_address, 0);
1179 }
1180 __ bind(L_done);
1181 }
1182
1183 #undef __
1184 #define __ masm.
1185
1186 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
1187 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1188
1189 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
1190 __ testb(gc_state_fast, ShenandoahThreadLocalData::gc_state_to_fast(test_state));
1191 __ jcc(Assembler::notZero, *entry());
1192 __ bind(*continuation());
1193 }
1194
1195 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1196 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1197
1198 __ bind(*entry());
1199
1200 Label L_done, L_barriers;
1201
1202 // If we need to load ourselves, do it here.
1203 if (_do_load) {
1204 if (_narrow) {
1205 __ movl(_obj, _addr);
1206 } else {
1207 __ movq(_obj, _addr);
1208 }
1209 }
1210
1211 // If the object is null, there is no point in applying barriers.
1212 if (_narrow) {
1251 // For encoding, dst can only turn null if we are dealing with weak loads.
1252 // Otherwise, we have already null-checked. We can skip all this if we performed
1253 // the load ourselves, which means the value is not used by caller.
1254 if (_narrow && !_do_load) {
1255 if (_needs_load_ref_weak_barrier) {
1256 __ encode_heap_oop(_obj);
1257 } else {
1258 __ encode_heap_oop_not_null(_obj);
1259 }
1260 }
1261 __ jmp(L_done);
1262 }
1263
1264 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Register obj, Register tmp1, Register tmp2) {
1265 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1266 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1267
1268 Label L_fast, L_done;
1269
1270 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1271 if (_needs_load_ref_barrier) {
1272 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1273 __ testb(gc_state, ShenandoahHeap::MARKING);
1274 __ jccb(Assembler::zero, L_done);
1275 }
1276
1277 // Check if buffer is already full. Go slow, if so.
1278 __ movptr(tmp1, index);
1279 __ testptr(tmp1, tmp1);
1280 __ jccb(Assembler::notZero, L_fast);
1281
1282 // Slow-path: call runtime to handle.
1283 {
1284 // Shuffle in the arguments. The end result should be:
1285 // c_rarg0 <-- obj
1286 //
1287 // Save clobbered registers before overwriting them.
1288 bool clobbered_c_rarg0 = false;
1289 if (c_rarg0 != obj) {
1290 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1291 __ mov(c_rarg0, obj);
1298 if (clobbered_c_rarg0) {
1299 pop_save_register(masm, c_rarg0);
1300 }
1301 __ jmpb(L_done);
1302 }
1303
1304 // Fast-path: put object into buffer.
1305 __ bind(L_fast);
1306 __ subptr(tmp1, wordSize);
1307 __ movptr(index, tmp1);
1308 __ addptr(tmp1, buffer);
1309 __ movptr(Address(tmp1, 0), obj);
1310
1311 __ bind(L_done);
1312 }
1313
1314 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm, Register obj, Address addr, Register tmp) {
1315 Label L_done;
1316
1317 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1318 if (_needs_keep_alive_barrier) {
1319 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1320 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0));
1321 __ jccb(Assembler::zero, L_done);
1322 }
1323
1324 // Weak/phantom loads are handled in slow path.
1325 if (!_needs_load_ref_weak_barrier) {
1326 // Compute the cset bitmap index
1327 __ movptr(tmp, obj);
1328 __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1329
1330 // If cset address is in good spot to just use it as offset. It almost always is.
1331 Address cset_addr_arg;
1332 intptr_t cset_addr = (intptr_t) ShenandoahHeap::in_cset_fast_test_addr();
1333 if ((cset_addr >> 3) < INT32_MAX) {
1334 assert(is_aligned(cset_addr, 8), "Sanity");
1335 cset_addr_arg = Address(tmp, checked_cast<int>(cset_addr >> 3), Address::times_8);
1336 } else {
1337 __ addptr(tmp, cset_addr);
1338 cset_addr_arg = Address(tmp, 0);
|
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 *
25 */
26
27 #include "gc/shenandoah/heuristics/shenandoahHeuristics.hpp"
28 #include "gc/shenandoah/mode/shenandoahMode.hpp"
29 #include "gc/shenandoah/shenandoahBarrierSet.hpp"
30 #include "gc/shenandoah/shenandoahBarrierSetAssembler.hpp"
31 #include "gc/shenandoah/shenandoahForwarding.hpp"
32 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
33 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
34 #include "gc/shenandoah/shenandoahRuntime.hpp"
35 #include "gc/shenandoah/shenandoahThreadLocalData.hpp"
36 #include "interpreter/interpreter.hpp"
37 #include "nativeInst_x86.hpp"
38 #include "runtime/javaThread.hpp"
39 #include "runtime/sharedRuntime.hpp"
40 #include "utilities/macros.hpp"
41 #ifdef COMPILER1
42 #include "c1/c1_LIRAssembler.hpp"
43 #include "c1/c1_MacroAssembler.hpp"
44 #include "gc/shenandoah/c1/shenandoahBarrierSetC1.hpp"
45 #endif
46 #ifdef COMPILER2
47 #include "gc/shenandoah/c2/shenandoahBarrierSetC2.hpp"
48 #endif
49
50 #define __ masm->
51
52 static void save_machine_state(MacroAssembler* masm, bool handle_gpr, bool handle_fp) {
53 if (handle_gpr) {
54 __ push_IU_state();
55 }
56
57 if (handle_fp) {
1003 } else {
1004 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_weak), c_rarg0, c_rarg1);
1005 }
1006 } else {
1007 assert(is_phantom, "only remaining strength");
1008 assert(is_native, "phantom must only be called off-heap");
1009 __ call_VM_leaf(CAST_FROM_FN_PTR(address, ShenandoahRuntime::load_reference_barrier_phantom), c_rarg0, c_rarg1);
1010 }
1011
1012 __ restore_live_registers_except_rax(true);
1013
1014 __ epilogue();
1015 }
1016
1017 #undef __
1018
1019 #endif // COMPILER1
1020
1021 #ifdef COMPILER2
1022
1023 address ShenandoahBarrierSetAssembler::parse_stub_address(address pc) {
1024 NativeInstruction* ni = nativeInstruction_at(pc);
1025 assert(ni->is_jump(), "Initial code version: GC barrier fastpath must be a jump");
1026 NativeJump* jmp = nativeJump_at(pc);
1027 return jmp->jump_destination();
1028 }
1029
1030 void insert_5_byte_nop(address pc) {
1031 *(pc + 0) = 0x0F;
1032 *(pc + 1) = 0x1F;
1033 *(pc + 2) = 0x44;
1034 *(pc + 3) = 0x00;
1035 *(pc + 4) = 0x00;
1036 ICache::invalidate_range(pc, 5);
1037 }
1038
1039 bool is_5_byte_nop(address pc) {
1040 if (*(pc + 0) != 0x0F) return false;
1041 if (*(pc + 1) != 0x1F) return false;
1042 if (*(pc + 2) != 0x44) return false;
1043 if (*(pc + 3) != 0x00) return false;
1044 if (*(pc + 4) != 0x00) return false;
1045 return true;
1046 }
1047
1048 void check_at(bool cond, address pc, const char* msg) {
1049 assert(cond, "%s: at PC " PTR_FORMAT ": %02x%02x%02x%02x%02x",
1050 msg, p2i(pc), *(pc + 0), *(pc + 1), *(pc + 2), *(pc + 3), *(pc + 4));
1051 }
1052
1053 void ShenandoahBarrierSetAssembler::patch_branch_to_nop(address pc) {
1054 NativeInstruction* ni = nativeInstruction_at(pc);
1055 if (ni->is_jump()) {
1056 insert_5_byte_nop(pc);
1057 } else {
1058 check_at(is_5_byte_nop(pc), pc, "Should already be nop");
1059 }
1060 }
1061
1062 void ShenandoahBarrierSetAssembler::patch_nop_to_branch(address pc, address stub_addr) {
1063 NativeInstruction* ni = nativeInstruction_at(pc);
1064 if (is_5_byte_nop(pc)) {
1065 NativeJump::insert(pc, stub_addr);
1066 } else {
1067 check_at(ni->is_jump(), pc, "Should already be jump");
1068 check_at(nativeJump_at(pc)->jump_destination() == stub_addr, pc, "Jump should be to the same address");
1069 }
1070 }
1071
1072 #undef __
1073 #define __ masm->
1074
1075 void ShenandoahBarrierSetAssembler::load_c2(const MachNode* node, MacroAssembler* masm, Register dst, Address src, bool narrow) {
1076 // Do the actual load. This load is the candidate for implicit null check, and MUST come first.
1077 if (narrow) {
1078 __ movl(dst, src);
1079 } else {
1080 __ movq(dst, src);
1081 }
1082
1083 // Post-barrier: LRB
1084 if (ShenandoahBarrierStubC2::needs_slow_barrier(node)) {
1085 ShenandoahBarrierStubC2* const stub = ShenandoahBarrierStubC2::create(node, dst, src, narrow, false);
1086 char check = 0;
1087 check |= ShenandoahBarrierStubC2::needs_keep_alive_barrier(node) ? ShenandoahHeap::MARKING : 0;
1088 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier(node) ? ShenandoahHeap::HAS_FORWARDED : 0;
1089 check |= ShenandoahBarrierStubC2::needs_load_ref_barrier_weak(node) ? ShenandoahHeap::WEAK_ROOTS : 0;
1090 stub->enter_if_gc_state(*masm, check);
1091 }
1219 assert(CardTable::dirty_card_val() == 0, "Encoding assumption");
1220 Label L_done;
1221 if (UseCondCardMark) {
1222 __ cmpb(card_address, 0);
1223 __ jccb(Assembler::equal, L_done);
1224 }
1225 if (UseCompressedOops && CompressedOops::base() == nullptr) {
1226 __ movb(card_address, r12);
1227 } else {
1228 __ movb(card_address, 0);
1229 }
1230 __ bind(L_done);
1231 }
1232
1233 #undef __
1234 #define __ masm.
1235
1236 void ShenandoahBarrierStubC2::enter_if_gc_state(MacroAssembler& masm, const char test_state) {
1237 Assembler::InlineSkippedInstructionsCounter skip_counter(&masm);
1238
1239 if (ShenandoahGCStateCheckRemove) {
1240 // Unrealistic: remove all barrier fastpath checks.
1241 } else if (ShenandoahGCStateCheckHotpatch) {
1242 // Emit the unconditional branch in the first version of the method.
1243 // Let the rest of runtime figure out how to manage it.
1244 __ relocate(barrier_Relocation::spec());
1245 __ jmp(*entry(), /* maybe_short = */ false);
1246
1247 Label L_done;
1248 #ifdef ASSERT
1249 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1250 __ testb(gc_state, 0xFF);
1251 __ jccb(Assembler::zero, L_done);
1252 __ hlt(); // Correctness bug: barrier is NOP-ed, but heap is NOT IDLE
1253 #endif
1254 __ bind(*continuation());
1255 // This is futile to assert, because barriers are turned off asynchronously.
1256 // #ifdef ASSERT
1257 // __ testb(gc_state, 0xFF);
1258 // __ jccb(Assembler::notZero, L_done);
1259 // __ hlt(); // Performance bug: barrier is JMP-ed, but heap is IDLE
1260 // #endif
1261 __ bind(L_done);
1262 } else {
1263 Address gc_state_fast(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_fast_offset()));
1264 __ testb(gc_state_fast, ShenandoahThreadLocalData::gc_state_to_fast(test_state));
1265 __ jcc(Assembler::notZero, *entry());
1266 __ bind(*continuation());
1267 }
1268 }
1269
1270 void ShenandoahBarrierStubC2::emit_code(MacroAssembler& masm) {
1271 assert(_needs_keep_alive_barrier || _needs_load_ref_barrier, "Why are you here?");
1272
1273 __ bind(*entry());
1274
1275 Label L_done, L_barriers;
1276
1277 // If we need to load ourselves, do it here.
1278 if (_do_load) {
1279 if (_narrow) {
1280 __ movl(_obj, _addr);
1281 } else {
1282 __ movq(_obj, _addr);
1283 }
1284 }
1285
1286 // If the object is null, there is no point in applying barriers.
1287 if (_narrow) {
1326 // For encoding, dst can only turn null if we are dealing with weak loads.
1327 // Otherwise, we have already null-checked. We can skip all this if we performed
1328 // the load ourselves, which means the value is not used by caller.
1329 if (_narrow && !_do_load) {
1330 if (_needs_load_ref_weak_barrier) {
1331 __ encode_heap_oop(_obj);
1332 } else {
1333 __ encode_heap_oop_not_null(_obj);
1334 }
1335 }
1336 __ jmp(L_done);
1337 }
1338
1339 void ShenandoahBarrierStubC2::keepalive(MacroAssembler& masm, Register obj, Register tmp1, Register tmp2) {
1340 Address index(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_index_offset()));
1341 Address buffer(r15_thread, in_bytes(ShenandoahThreadLocalData::satb_mark_queue_buffer_offset()));
1342
1343 Label L_fast, L_done;
1344
1345 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1346 // Hotpatched GC checks only care about idle/non-idle state, so needs a check anyhow.
1347 if (_needs_load_ref_barrier || ShenandoahGCStateCheckHotpatch) {
1348 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1349 __ testb(gc_state, ShenandoahHeap::MARKING);
1350 __ jccb(Assembler::zero, L_done);
1351 }
1352
1353 // Check if buffer is already full. Go slow, if so.
1354 __ movptr(tmp1, index);
1355 __ testptr(tmp1, tmp1);
1356 __ jccb(Assembler::notZero, L_fast);
1357
1358 // Slow-path: call runtime to handle.
1359 {
1360 // Shuffle in the arguments. The end result should be:
1361 // c_rarg0 <-- obj
1362 //
1363 // Save clobbered registers before overwriting them.
1364 bool clobbered_c_rarg0 = false;
1365 if (c_rarg0 != obj) {
1366 clobbered_c_rarg0 = push_save_register_if_live(masm, c_rarg0);
1367 __ mov(c_rarg0, obj);
1374 if (clobbered_c_rarg0) {
1375 pop_save_register(masm, c_rarg0);
1376 }
1377 __ jmpb(L_done);
1378 }
1379
1380 // Fast-path: put object into buffer.
1381 __ bind(L_fast);
1382 __ subptr(tmp1, wordSize);
1383 __ movptr(index, tmp1);
1384 __ addptr(tmp1, buffer);
1385 __ movptr(Address(tmp1, 0), obj);
1386
1387 __ bind(L_done);
1388 }
1389
1390 void ShenandoahBarrierStubC2::lrb(MacroAssembler& masm, Register obj, Address addr, Register tmp) {
1391 Label L_done;
1392
1393 // If another barrier is enabled as well, do a runtime check for a specific barrier.
1394 // Hotpatched GC checks only care about idle/non-idle state, so needs a check anyhow.
1395 if (_needs_keep_alive_barrier || ShenandoahGCStateCheckHotpatch) {
1396 Address gc_state(r15_thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
1397 __ testb(gc_state, ShenandoahHeap::HAS_FORWARDED | (_needs_load_ref_weak_barrier ? ShenandoahHeap::WEAK_ROOTS : 0));
1398 __ jccb(Assembler::zero, L_done);
1399 }
1400
1401 // Weak/phantom loads are handled in slow path.
1402 if (!_needs_load_ref_weak_barrier) {
1403 // Compute the cset bitmap index
1404 __ movptr(tmp, obj);
1405 __ shrptr(tmp, ShenandoahHeapRegion::region_size_bytes_shift_jint());
1406
1407 // If cset address is in good spot to just use it as offset. It almost always is.
1408 Address cset_addr_arg;
1409 intptr_t cset_addr = (intptr_t) ShenandoahHeap::in_cset_fast_test_addr();
1410 if ((cset_addr >> 3) < INT32_MAX) {
1411 assert(is_aligned(cset_addr, 8), "Sanity");
1412 cset_addr_arg = Address(tmp, checked_cast<int>(cset_addr >> 3), Address::times_8);
1413 } else {
1414 __ addptr(tmp, cset_addr);
1415 cset_addr_arg = Address(tmp, 0);
|