40 #include "runtime/safepointMechanism.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "runtime/thread.inline.hpp"
43 #include "utilities/powerOfTwo.hpp"
44
45 // Implementation of InterpreterMacroAssembler
46
47 void InterpreterMacroAssembler::jump_to_entry(address entry) {
48 assert(entry, "Entry must have been generated by now");
49 jump(RuntimeAddress(entry));
50 }
51
52 void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {
53 Label update, next, none;
54
55 interp_verify_oop(obj, atos);
56
57 testptr(obj, obj);
58 jccb(Assembler::notZero, update);
59 orptr(mdo_addr, TypeEntries::null_seen);
60 jmpb(next);
61
62 bind(update);
63 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
64 load_klass(obj, obj, tmp_load_klass);
65
66 xorptr(obj, mdo_addr);
67 testptr(obj, TypeEntries::type_klass_mask);
68 jccb(Assembler::zero, next); // klass seen before, nothing to
69 // do. The unknown bit may have been
70 // set already but no need to check.
71
72 testptr(obj, TypeEntries::type_unknown);
73 jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
74
75 cmpptr(mdo_addr, 0);
76 jccb(Assembler::equal, none);
77 cmpptr(mdo_addr, TypeEntries::null_seen);
78 jccb(Assembler::equal, none);
79 // There is a chance that the checks above (re-reading profiling
80 // data from memory) fail if another thread has just set the
1214 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1215 const int mark_offset = lock_offset +
1216 BasicLock::displaced_header_offset_in_bytes();
1217
1218 Label slow_case;
1219
1220 // Load object pointer into obj_reg
1221 movptr(obj_reg, Address(lock_reg, obj_offset));
1222
1223 if (DiagnoseSyncOnValueBasedClasses != 0) {
1224 load_klass(tmp_reg, obj_reg, rklass_decode_tmp);
1225 movl(tmp_reg, Address(tmp_reg, Klass::access_flags_offset()));
1226 testl(tmp_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
1227 jcc(Assembler::notZero, slow_case);
1228 }
1229
1230 if (UseBiasedLocking) {
1231 biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp_reg, rklass_decode_tmp, false, done, &slow_case);
1232 }
1233
1234 // Load immediate 1 into swap_reg %rax
1235 movl(swap_reg, (int32_t)1);
1236
1237 // Load (object->mark() | 1) into swap_reg %rax
1238 orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1239
1240 // Save (object->mark() | 1) into BasicLock's displaced header
1241 movptr(Address(lock_reg, mark_offset), swap_reg);
1242
1243 assert(lock_offset == 0,
1244 "displaced header must be first word in BasicObjectLock");
1245
1246 lock();
1247 cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1248 if (PrintBiasedLockingStatistics) {
1249 cond_inc32(Assembler::zero,
1250 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
1251 }
1252 jcc(Assembler::zero, done);
1253
1254 const int zero_bits = LP64_ONLY(7) NOT_LP64(3);
1255
1256 // Fast check for recursive lock.
1257 //
1258 // Can apply the optimization only if this is a stack lock
1259 // allocated in this thread. For efficiency, we can focus on
1260 // recently allocated stack locks (instead of reading the stack
1261 // base and checking whether 'mark' points inside the current
1262 // thread stack):
1263 // 1) (mark & zero_bits) == 0, and
1264 // 2) rsp <= mark < mark + os::pagesize()
1265 //
1266 // Warning: rsp + os::pagesize can overflow the stack base. We must
1267 // neither apply the optimization for an inflated lock allocated
1268 // just above the thread stack (this is why condition 1 matters)
1269 // nor apply the optimization if the stack lock is inside the stack
1270 // of another thread. The latter is avoided even in case of overflow
1271 // because we have guard pages at the end of all stacks. Hence, if
1272 // we go over the stack base and hit the stack of another thread,
1273 // this should not be in a writeable area that could contain a
1274 // stack lock allocated by that thread. As a consequence, a stack
1275 // lock less than page size away from rsp is guaranteed to be
1276 // owned by the current thread.
1277 //
1278 // These 3 tests can be done by evaluating the following
1279 // expression: ((mark - rsp) & (zero_bits - os::vm_page_size())),
1280 // assuming both stack pointer and pagesize have their
1281 // least significant bits clear.
1282 // NOTE: the mark is in swap_reg %rax as the result of cmpxchg
1283 subptr(swap_reg, rsp);
1284 andptr(swap_reg, zero_bits - os::vm_page_size());
1285
1286 // Save the test result, for recursive case, the result is zero
1287 movptr(Address(lock_reg, mark_offset), swap_reg);
1288
1289 if (PrintBiasedLockingStatistics) {
1290 cond_inc32(Assembler::zero,
1291 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
1292 }
1293 jcc(Assembler::zero, done);
1294
1295 bind(slow_case);
1296
1297 // Call the runtime routine for slow case
1298 call_VM(noreg,
1299 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
1300 lock_reg);
1301
1302 bind(done);
1303 }
1304 }
1305
1306
1307 // Unlocks an object. Used in monitorexit bytecode and
1308 // remove_activation. Throws an IllegalMonitorException if object is
1309 // not locked by current thread.
1310 //
1311 // Args:
1312 // rdx, c_rarg1: BasicObjectLock for lock
1313 //
1314 // Kills:
1315 // rax
1316 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
1317 // rscratch1 (scratch reg)
1318 // rax, rbx, rcx, rdx
1319 void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
1320 assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
1321 "The argument is only for looks. It must be c_rarg1");
1322
1323 if (UseHeavyMonitors) {
1324 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1325 } else {
1326 Label done;
1327
1328 const Register swap_reg = rax; // Must use rax for cmpxchg instruction
1329 const Register header_reg = LP64_ONLY(c_rarg2) NOT_LP64(rbx); // Will contain the old oopMark
1330 const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
1331
1332 save_bcp(); // Save in case of exception
1333
1334 // Convert from BasicObjectLock structure to object and BasicLock
1335 // structure Store the BasicLock address into %rax
1336 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
1337
1338 // Load oop into obj_reg(%c_rarg3)
1339 movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
1340
1341 // Free entry
1342 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), (int32_t)NULL_WORD);
1343
1344 if (UseBiasedLocking) {
1345 biased_locking_exit(obj_reg, header_reg, done);
1346 }
1347
1348 // Load the old header from BasicLock structure
1349 movptr(header_reg, Address(swap_reg,
1350 BasicLock::displaced_header_offset_in_bytes()));
1351
1352 // Test for recursion
1353 testptr(header_reg, header_reg);
1354
1355 // zero for recursive case
1356 jcc(Assembler::zero, done);
1357
1358 // Atomic swap back the old header
1359 lock();
1360 cmpxchgptr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1361
1362 // zero for simple unlock of a stack-lock case
1363 jcc(Assembler::zero, done);
1364
1365
1366 // Call the runtime routine for slow case.
1367 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), obj_reg); // restore obj
1368 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1369
1370 bind(done);
1371
1372 restore_bcp();
1373 }
1374 }
1375
1376 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
1377 Label& zero_continue) {
1378 assert(ProfileInterpreter, "must be profiling interpreter");
1379 movptr(mdp, Address(rbp, frame::interpreter_frame_mdp_offset * wordSize));
1380 testptr(mdp, mdp);
1381 jcc(Assembler::zero, zero_continue);
1382 }
1383
1384
1385 // Set the method data pointer for the current bcp.
|
40 #include "runtime/safepointMechanism.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "runtime/thread.inline.hpp"
43 #include "utilities/powerOfTwo.hpp"
44
45 // Implementation of InterpreterMacroAssembler
46
47 void InterpreterMacroAssembler::jump_to_entry(address entry) {
48 assert(entry, "Entry must have been generated by now");
49 jump(RuntimeAddress(entry));
50 }
51
52 void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {
53 Label update, next, none;
54
55 interp_verify_oop(obj, atos);
56
57 testptr(obj, obj);
58 jccb(Assembler::notZero, update);
59 orptr(mdo_addr, TypeEntries::null_seen);
60 jmp(next);
61
62 bind(update);
63 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
64 load_klass(obj, obj, tmp_load_klass);
65
66 xorptr(obj, mdo_addr);
67 testptr(obj, TypeEntries::type_klass_mask);
68 jccb(Assembler::zero, next); // klass seen before, nothing to
69 // do. The unknown bit may have been
70 // set already but no need to check.
71
72 testptr(obj, TypeEntries::type_unknown);
73 jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
74
75 cmpptr(mdo_addr, 0);
76 jccb(Assembler::equal, none);
77 cmpptr(mdo_addr, TypeEntries::null_seen);
78 jccb(Assembler::equal, none);
79 // There is a chance that the checks above (re-reading profiling
80 // data from memory) fail if another thread has just set the
1214 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1215 const int mark_offset = lock_offset +
1216 BasicLock::displaced_header_offset_in_bytes();
1217
1218 Label slow_case;
1219
1220 // Load object pointer into obj_reg
1221 movptr(obj_reg, Address(lock_reg, obj_offset));
1222
1223 if (DiagnoseSyncOnValueBasedClasses != 0) {
1224 load_klass(tmp_reg, obj_reg, rklass_decode_tmp);
1225 movl(tmp_reg, Address(tmp_reg, Klass::access_flags_offset()));
1226 testl(tmp_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
1227 jcc(Assembler::notZero, slow_case);
1228 }
1229
1230 if (UseBiasedLocking) {
1231 biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp_reg, rklass_decode_tmp, false, done, &slow_case);
1232 }
1233
1234 if (UseFastLocking) {
1235 #ifdef _LP64
1236 const Register thread = r15_thread;
1237 #else
1238 const Register thread = lock_reg;
1239 get_thread(thread);
1240 #endif
1241 // Load object header, prepare for CAS from unlocked to locked.
1242 movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1243 fast_lock_impl(obj_reg, swap_reg, thread, tmp_reg, slow_case);
1244 jmp(done);
1245 } else {
1246 // Load immediate 1 into swap_reg %rax
1247 movl(swap_reg, (int32_t)1);
1248
1249 // Load (object->mark() | 1) into swap_reg %rax
1250 orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1251
1252 // Save (object->mark() | 1) into BasicLock's displaced header
1253 movptr(Address(lock_reg, mark_offset), swap_reg);
1254
1255 assert(lock_offset == 0,
1256 "displaced header must be first word in BasicObjectLock");
1257
1258 lock();
1259 cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1260 if (PrintBiasedLockingStatistics) {
1261 cond_inc32(Assembler::zero,
1262 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
1263 }
1264 jcc(Assembler::zero, done);
1265
1266 const int zero_bits = LP64_ONLY(7) NOT_LP64(3);
1267
1268 // Fast check for recursive lock.
1269 //
1270 // Can apply the optimization only if this is a stack lock
1271 // allocated in this thread. For efficiency, we can focus on
1272 // recently allocated stack locks (instead of reading the stack
1273 // base and checking whether 'mark' points inside the current
1274 // thread stack):
1275 // 1) (mark & zero_bits) == 0, and
1276 // 2) rsp <= mark < mark + os::pagesize()
1277 //
1278 // Warning: rsp + os::pagesize can overflow the stack base. We must
1279 // neither apply the optimization for an inflated lock allocated
1280 // just above the thread stack (this is why condition 1 matters)
1281 // nor apply the optimization if the stack lock is inside the stack
1282 // of another thread. The latter is avoided even in case of overflow
1283 // because we have guard pages at the end of all stacks. Hence, if
1284 // we go over the stack base and hit the stack of another thread,
1285 // this should not be in a writeable area that could contain a
1286 // stack lock allocated by that thread. As a consequence, a stack
1287 // lock less than page size away from rsp is guaranteed to be
1288 // owned by the current thread.
1289 //
1290 // These 3 tests can be done by evaluating the following
1291 // expression: ((mark - rsp) & (zero_bits - os::vm_page_size())),
1292 // assuming both stack pointer and pagesize have their
1293 // least significant bits clear.
1294 // NOTE: the mark is in swap_reg %rax as the result of cmpxchg
1295 subptr(swap_reg, rsp);
1296 andptr(swap_reg, zero_bits - os::vm_page_size());
1297
1298 // Save the test result, for recursive case, the result is zero
1299 movptr(Address(lock_reg, mark_offset), swap_reg);
1300
1301 if (PrintBiasedLockingStatistics) {
1302 cond_inc32(Assembler::zero,
1303 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
1304 }
1305 jcc(Assembler::zero, done);
1306 }
1307 bind(slow_case);
1308
1309 // Call the runtime routine for slow case
1310 call_VM(noreg,
1311 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
1312 UseFastLocking ? obj_reg : lock_reg);
1313
1314 bind(done);
1315 }
1316 }
1317
1318
1319 // Unlocks an object. Used in monitorexit bytecode and
1320 // remove_activation. Throws an IllegalMonitorException if object is
1321 // not locked by current thread.
1322 //
1323 // Args:
1324 // rdx, c_rarg1: BasicObjectLock for lock
1325 //
1326 // Kills:
1327 // rax
1328 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
1329 // rscratch1 (scratch reg)
1330 // rax, rbx, rcx, rdx
1331 void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
1332 assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
1333 "The argument is only for looks. It must be c_rarg1");
1334
1335 if (UseHeavyMonitors) {
1336 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1337 } else {
1338 Label done, slow_case;
1339
1340 const Register swap_reg = rax; // Must use rax for cmpxchg instruction
1341 const Register header_reg = LP64_ONLY(c_rarg2) NOT_LP64(rbx); // Will contain the old oopMark
1342 const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
1343
1344 save_bcp(); // Save in case of exception
1345
1346 // Load oop into obj_reg(%c_rarg3)
1347 movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
1348
1349 // Free entry
1350 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), (int32_t)NULL_WORD);
1351
1352 if (UseFastLocking) {
1353 #ifdef _LP64
1354 const Register thread = r15_thread;
1355 #else
1356 const Register thread = header_reg;
1357 get_thread(thread);
1358 #endif
1359 // Handle unstructured locking.
1360 Register tmp = swap_reg;
1361 movptr(tmp, Address(thread, JavaThread::lock_stack_current_offset()));
1362 cmpptr(obj_reg, Address(tmp, -oopSize));
1363 jcc(Assembler::notEqual, slow_case);
1364 // Try to swing header from locked to unlock.
1365 movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1366 andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
1367 fast_unlock_impl(obj_reg, swap_reg, header_reg, slow_case);
1368 jmp(done);
1369 } else {
1370 // Convert from BasicObjectLock structure to object and BasicLock
1371 // structure Store the BasicLock address into %rax
1372 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
1373
1374 if (UseBiasedLocking) {
1375 biased_locking_exit(obj_reg, header_reg, done);
1376 }
1377
1378 // Load the old header from BasicLock structure
1379 movptr(header_reg, Address(swap_reg,
1380 BasicLock::displaced_header_offset_in_bytes()));
1381
1382 // Test for recursion
1383 testptr(header_reg, header_reg);
1384
1385 // zero for recursive case
1386 jcc(Assembler::zero, done);
1387
1388 // Atomic swap back the old header
1389 lock();
1390 cmpxchgptr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1391
1392 // zero for simple unlock of a stack-lock case
1393 jcc(Assembler::zero, done);
1394 }
1395
1396 bind(slow_case);
1397 // Call the runtime routine for slow case.
1398 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), obj_reg); // restore obj
1399 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1400
1401 bind(done);
1402
1403 restore_bcp();
1404 }
1405 }
1406
1407 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
1408 Label& zero_continue) {
1409 assert(ProfileInterpreter, "must be profiling interpreter");
1410 movptr(mdp, Address(rbp, frame::interpreter_frame_mdp_offset * wordSize));
1411 testptr(mdp, mdp);
1412 jcc(Assembler::zero, zero_continue);
1413 }
1414
1415
1416 // Set the method data pointer for the current bcp.
|