1206 const Register swap_reg = rax; // Must use rax for cmpxchg instruction
1207 const Register tmp_reg = rbx;
1208 const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
1209 const Register rklass_decode_tmp = rscratch1;
1210
1211 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
1212 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1213 const int mark_offset = lock_offset +
1214 BasicLock::displaced_header_offset_in_bytes();
1215
1216 // Load object pointer into obj_reg
1217 movptr(obj_reg, Address(lock_reg, obj_offset));
1218
1219 if (DiagnoseSyncOnValueBasedClasses != 0) {
1220 load_klass(tmp_reg, obj_reg, rklass_decode_tmp);
1221 movl(tmp_reg, Address(tmp_reg, Klass::access_flags_offset()));
1222 testl(tmp_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
1223 jcc(Assembler::notZero, slow_case);
1224 }
1225
1226 // Load immediate 1 into swap_reg %rax
1227 movl(swap_reg, 1);
1228
1229 // Load (object->mark() | 1) into swap_reg %rax
1230 orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1231
1232 // Save (object->mark() | 1) into BasicLock's displaced header
1233 movptr(Address(lock_reg, mark_offset), swap_reg);
1234
1235 assert(lock_offset == 0,
1236 "displaced header must be first word in BasicObjectLock");
1237
1238 lock();
1239 cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1240 jcc(Assembler::zero, count_locking);
1241
1242 const int zero_bits = LP64_ONLY(7) NOT_LP64(3);
1243
1244 // Fast check for recursive lock.
1245 //
1246 // Can apply the optimization only if this is a stack lock
1247 // allocated in this thread. For efficiency, we can focus on
1248 // recently allocated stack locks (instead of reading the stack
1249 // base and checking whether 'mark' points inside the current
1250 // thread stack):
1251 // 1) (mark & zero_bits) == 0, and
1252 // 2) rsp <= mark < mark + os::pagesize()
1253 //
1254 // Warning: rsp + os::pagesize can overflow the stack base. We must
1255 // neither apply the optimization for an inflated lock allocated
1256 // just above the thread stack (this is why condition 1 matters)
1257 // nor apply the optimization if the stack lock is inside the stack
1258 // of another thread. The latter is avoided even in case of overflow
1259 // because we have guard pages at the end of all stacks. Hence, if
1260 // we go over the stack base and hit the stack of another thread,
1261 // this should not be in a writeable area that could contain a
1262 // stack lock allocated by that thread. As a consequence, a stack
1263 // lock less than page size away from rsp is guaranteed to be
1264 // owned by the current thread.
1265 //
1266 // These 3 tests can be done by evaluating the following
1267 // expression: ((mark - rsp) & (zero_bits - os::vm_page_size())),
1268 // assuming both stack pointer and pagesize have their
1269 // least significant bits clear.
1270 // NOTE: the mark is in swap_reg %rax as the result of cmpxchg
1271 subptr(swap_reg, rsp);
1272 andptr(swap_reg, zero_bits - (int)os::vm_page_size());
1273
1274 // Save the test result, for recursive case, the result is zero
1275 movptr(Address(lock_reg, mark_offset), swap_reg);
1276 jcc(Assembler::notZero, slow_case);
1277
1278 bind(count_locking);
1279 inc_held_monitor_count();
1280 jmp(done);
1281
1282 bind(slow_case);
1283
1284 // Call the runtime routine for slow case
1285 call_VM(noreg,
1286 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
1287 lock_reg);
1288
1289 bind(done);
1290 }
1291 }
1292
1293
1294 // Unlocks an object. Used in monitorexit bytecode and
1295 // remove_activation. Throws an IllegalMonitorException if object is
1296 // not locked by current thread.
1297 //
1298 // Args:
1299 // rdx, c_rarg1: BasicObjectLock for lock
1300 //
1301 // Kills:
1302 // rax
1303 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
1304 // rscratch1 (scratch reg)
1305 // rax, rbx, rcx, rdx
1306 void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
1307 assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
1308 "The argument is only for looks. It must be c_rarg1");
1309
1310 if (UseHeavyMonitors) {
1311 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1312 } else {
1313 Label count_locking, done, slow_case;
1314
1315 const Register swap_reg = rax; // Must use rax for cmpxchg instruction
1316 const Register header_reg = LP64_ONLY(c_rarg2) NOT_LP64(rbx); // Will contain the old oopMark
1317 const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
1318
1319 save_bcp(); // Save in case of exception
1320
1321 // Convert from BasicObjectLock structure to object and BasicLock
1322 // structure Store the BasicLock address into %rax
1323 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
1324
1325 // Load oop into obj_reg(%c_rarg3)
1326 movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
1327
1328 // Free entry
1329 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), NULL_WORD);
1330
1331 // Load the old header from BasicLock structure
1332 movptr(header_reg, Address(swap_reg,
1333 BasicLock::displaced_header_offset_in_bytes()));
1334
1335 // Test for recursion
1336 testptr(header_reg, header_reg);
1337
1338 // zero for recursive case
1339 jcc(Assembler::zero, count_locking);
1340
1341 // Atomic swap back the old header
1342 lock();
1343 cmpxchgptr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1344
1345 // zero for simple unlock of a stack-lock case
1346 jcc(Assembler::notZero, slow_case);
1347
1348 bind(count_locking);
1349 dec_held_monitor_count();
1350 jmp(done);
1351
1352 bind(slow_case);
1353 // Call the runtime routine for slow case.
1354 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), obj_reg); // restore obj
1355 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1356
1357 bind(done);
1358
1359 restore_bcp();
1360 }
1361 }
1362
1363 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
1364 Label& zero_continue) {
1365 assert(ProfileInterpreter, "must be profiling interpreter");
1366 movptr(mdp, Address(rbp, frame::interpreter_frame_mdp_offset * wordSize));
1367 testptr(mdp, mdp);
1368 jcc(Assembler::zero, zero_continue);
|
1206 const Register swap_reg = rax; // Must use rax for cmpxchg instruction
1207 const Register tmp_reg = rbx;
1208 const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
1209 const Register rklass_decode_tmp = rscratch1;
1210
1211 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
1212 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1213 const int mark_offset = lock_offset +
1214 BasicLock::displaced_header_offset_in_bytes();
1215
1216 // Load object pointer into obj_reg
1217 movptr(obj_reg, Address(lock_reg, obj_offset));
1218
1219 if (DiagnoseSyncOnValueBasedClasses != 0) {
1220 load_klass(tmp_reg, obj_reg, rklass_decode_tmp);
1221 movl(tmp_reg, Address(tmp_reg, Klass::access_flags_offset()));
1222 testl(tmp_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
1223 jcc(Assembler::notZero, slow_case);
1224 }
1225
1226 if (UseFastLocking) {
1227 #ifdef _LP64
1228 const Register thread = r15_thread;
1229 #else
1230 const Register thread = lock_reg;
1231 get_thread(thread);
1232 #endif
1233 // Load object header, prepare for CAS from unlocked to locked.
1234 movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1235 fast_lock_impl(obj_reg, swap_reg, thread, tmp_reg, slow_case);
1236 } else {
1237 // Load immediate 1 into swap_reg %rax
1238 movl(swap_reg, 1);
1239
1240 // Load (object->mark() | 1) into swap_reg %rax
1241 orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1242
1243 // Save (object->mark() | 1) into BasicLock's displaced header
1244 movptr(Address(lock_reg, mark_offset), swap_reg);
1245
1246 assert(lock_offset == 0,
1247 "displaced header must be first word in BasicObjectLock");
1248
1249 lock();
1250 cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1251 jcc(Assembler::zero, count_locking);
1252
1253 const int zero_bits = LP64_ONLY(7) NOT_LP64(3);
1254
1255 // Fast check for recursive lock.
1256 //
1257 // Can apply the optimization only if this is a stack lock
1258 // allocated in this thread. For efficiency, we can focus on
1259 // recently allocated stack locks (instead of reading the stack
1260 // base and checking whether 'mark' points inside the current
1261 // thread stack):
1262 // 1) (mark & zero_bits) == 0, and
1263 // 2) rsp <= mark < mark + os::pagesize()
1264 //
1265 // Warning: rsp + os::pagesize can overflow the stack base. We must
1266 // neither apply the optimization for an inflated lock allocated
1267 // just above the thread stack (this is why condition 1 matters)
1268 // nor apply the optimization if the stack lock is inside the stack
1269 // of another thread. The latter is avoided even in case of overflow
1270 // because we have guard pages at the end of all stacks. Hence, if
1271 // we go over the stack base and hit the stack of another thread,
1272 // this should not be in a writeable area that could contain a
1273 // stack lock allocated by that thread. As a consequence, a stack
1274 // lock less than page size away from rsp is guaranteed to be
1275 // owned by the current thread.
1276 //
1277 // These 3 tests can be done by evaluating the following
1278 // expression: ((mark - rsp) & (zero_bits - os::vm_page_size())),
1279 // assuming both stack pointer and pagesize have their
1280 // least significant bits clear.
1281 // NOTE: the mark is in swap_reg %rax as the result of cmpxchg
1282 subptr(swap_reg, rsp);
1283 andptr(swap_reg, zero_bits - (int)os::vm_page_size());
1284
1285 // Save the test result, for recursive case, the result is zero
1286 movptr(Address(lock_reg, mark_offset), swap_reg);
1287 jcc(Assembler::notZero, slow_case);
1288
1289 bind(count_locking);
1290 }
1291 inc_held_monitor_count();
1292 jmp(done);
1293
1294 bind(slow_case);
1295
1296 // Call the runtime routine for slow case
1297 call_VM(noreg,
1298 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
1299 UseFastLocking ? obj_reg : lock_reg);
1300
1301 bind(done);
1302 }
1303 }
1304
1305
1306 // Unlocks an object. Used in monitorexit bytecode and
1307 // remove_activation. Throws an IllegalMonitorException if object is
1308 // not locked by current thread.
1309 //
1310 // Args:
1311 // rdx, c_rarg1: BasicObjectLock for lock
1312 //
1313 // Kills:
1314 // rax
1315 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
1316 // rscratch1 (scratch reg)
1317 // rax, rbx, rcx, rdx
1318 void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
1319 assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
1320 "The argument is only for looks. It must be c_rarg1");
1321
1322 if (UseHeavyMonitors) {
1323 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1324 } else {
1325 Label count_locking, done, slow_case;
1326
1327 const Register swap_reg = rax; // Must use rax for cmpxchg instruction
1328 const Register header_reg = LP64_ONLY(c_rarg2) NOT_LP64(rbx); // Will contain the old oopMark
1329 const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
1330
1331 save_bcp(); // Save in case of exception
1332
1333 if (!UseFastLocking) {
1334 // Convert from BasicObjectLock structure to object and BasicLock
1335 // structure Store the BasicLock address into %rax
1336 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
1337 }
1338
1339 // Load oop into obj_reg(%c_rarg3)
1340 movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
1341
1342 // Free entry
1343 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), NULL_WORD);
1344
1345 if (UseFastLocking) {
1346 #ifdef _LP64
1347 const Register thread = r15_thread;
1348 #else
1349 const Register thread = header_reg;
1350 get_thread(thread);
1351 #endif
1352 // Handle unstructured locking.
1353 Register tmp = swap_reg;
1354 movptr(tmp, Address(thread, JavaThread::lock_stack_current_offset()));
1355 cmpptr(obj_reg, Address(tmp, -oopSize));
1356 jcc(Assembler::notEqual, slow_case);
1357 // Try to swing header from locked to unlock.
1358 movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1359 andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
1360 fast_unlock_impl(obj_reg, swap_reg, header_reg, slow_case);
1361 } else {
1362 // Load the old header from BasicLock structure
1363 movptr(header_reg, Address(swap_reg,
1364 BasicLock::displaced_header_offset_in_bytes()));
1365
1366 // Test for recursion
1367 testptr(header_reg, header_reg);
1368
1369 // zero for recursive case
1370 jcc(Assembler::zero, count_locking);
1371
1372 // Atomic swap back the old header
1373 lock();
1374 cmpxchgptr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1375
1376 // zero for simple unlock of a stack-lock case
1377 jcc(Assembler::notZero, slow_case);
1378
1379 bind(count_locking);
1380 }
1381 dec_held_monitor_count();
1382 jmp(done);
1383
1384 bind(slow_case);
1385 // Call the runtime routine for slow case.
1386 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), obj_reg); // restore obj
1387 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1388
1389 bind(done);
1390
1391 restore_bcp();
1392 }
1393 }
1394
1395 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
1396 Label& zero_continue) {
1397 assert(ProfileInterpreter, "must be profiling interpreter");
1398 movptr(mdp, Address(rbp, frame::interpreter_frame_mdp_offset * wordSize));
1399 testptr(mdp, mdp);
1400 jcc(Assembler::zero, zero_continue);
|