40 #include "runtime/safepointMechanism.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "runtime/thread.inline.hpp"
43 #include "utilities/powerOfTwo.hpp"
44
45 // Implementation of InterpreterMacroAssembler
46
47 void InterpreterMacroAssembler::jump_to_entry(address entry) {
48 assert(entry, "Entry must have been generated by now");
49 jump(RuntimeAddress(entry));
50 }
51
52 void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {
53 Label update, next, none;
54
55 interp_verify_oop(obj, atos);
56
57 testptr(obj, obj);
58 jccb(Assembler::notZero, update);
59 orptr(mdo_addr, TypeEntries::null_seen);
60 jmpb(next);
61
62 bind(update);
63 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
64 load_klass(obj, obj, tmp_load_klass);
65
66 xorptr(obj, mdo_addr);
67 testptr(obj, TypeEntries::type_klass_mask);
68 jccb(Assembler::zero, next); // klass seen before, nothing to
69 // do. The unknown bit may have been
70 // set already but no need to check.
71
72 testptr(obj, TypeEntries::type_unknown);
73 jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
74
75 cmpptr(mdo_addr, 0);
76 jccb(Assembler::equal, none);
77 cmpptr(mdo_addr, TypeEntries::null_seen);
78 jccb(Assembler::equal, none);
79 // There is a chance that the checks above (re-reading profiling
80 // data from memory) fail if another thread has just set the
1180 call_VM(noreg, CAST_FROM_FN_PTR(address,
1181 InterpreterRuntime::build_method_counters), method);
1182 movptr(mcs, Address(method,Method::method_counters_offset()));
1183 testptr(mcs, mcs);
1184 jcc(Assembler::zero, skip); // No MethodCounters allocated, OutOfMemory
1185 bind(has_counters);
1186 }
1187
1188
1189 // Lock object
1190 //
1191 // Args:
1192 // rdx, c_rarg1: BasicObjectLock to be used for locking
1193 //
1194 // Kills:
1195 // rax, rbx
1196 void InterpreterMacroAssembler::lock_object(Register lock_reg) {
1197 assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
1198 "The argument is only for looks. It must be c_rarg1");
1199
1200 if (UseHeavyMonitors) {
1201 call_VM(noreg,
1202 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
1203 lock_reg);
1204 } else {
1205 Label done;
1206
1207 const Register swap_reg = rax; // Must use rax for cmpxchg instruction
1208 const Register tmp_reg = rbx; // Will be passed to biased_locking_enter to avoid a
1209 // problematic case where tmp_reg = no_reg.
1210 const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
1211 const Register rklass_decode_tmp = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1212
1213 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
1214 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1215 const int mark_offset = lock_offset +
1216 BasicLock::displaced_header_offset_in_bytes();
1217
1218 Label slow_case;
1219
1220 // Load object pointer into obj_reg
1221 movptr(obj_reg, Address(lock_reg, obj_offset));
1222
1223 if (DiagnoseSyncOnValueBasedClasses != 0) {
1224 load_klass(tmp_reg, obj_reg, rklass_decode_tmp);
1225 movl(tmp_reg, Address(tmp_reg, Klass::access_flags_offset()));
1226 testl(tmp_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
1227 jcc(Assembler::notZero, slow_case);
1228 }
1229
1230 if (UseBiasedLocking) {
1231 biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp_reg, rklass_decode_tmp, false, done, &slow_case);
1232 }
1233
1234 // Load immediate 1 into swap_reg %rax
1235 movl(swap_reg, (int32_t)1);
1236
1237 // Load (object->mark() | 1) into swap_reg %rax
1238 orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1239
1240 // Save (object->mark() | 1) into BasicLock's displaced header
1241 movptr(Address(lock_reg, mark_offset), swap_reg);
1242
1243 assert(lock_offset == 0,
1244 "displaced header must be first word in BasicObjectLock");
1245
1246 lock();
1247 cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1248 if (PrintBiasedLockingStatistics) {
1249 cond_inc32(Assembler::zero,
1250 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
1251 }
1252 jcc(Assembler::zero, done);
1253
1254 const int zero_bits = LP64_ONLY(7) NOT_LP64(3);
1255
1256 // Fast check for recursive lock.
1257 //
1258 // Can apply the optimization only if this is a stack lock
1259 // allocated in this thread. For efficiency, we can focus on
1260 // recently allocated stack locks (instead of reading the stack
1261 // base and checking whether 'mark' points inside the current
1262 // thread stack):
1263 // 1) (mark & zero_bits) == 0, and
1264 // 2) rsp <= mark < mark + os::pagesize()
1265 //
1266 // Warning: rsp + os::pagesize can overflow the stack base. We must
1267 // neither apply the optimization for an inflated lock allocated
1268 // just above the thread stack (this is why condition 1 matters)
1269 // nor apply the optimization if the stack lock is inside the stack
1270 // of another thread. The latter is avoided even in case of overflow
1271 // because we have guard pages at the end of all stacks. Hence, if
1272 // we go over the stack base and hit the stack of another thread,
1273 // this should not be in a writeable area that could contain a
1274 // stack lock allocated by that thread. As a consequence, a stack
1275 // lock less than page size away from rsp is guaranteed to be
1276 // owned by the current thread.
1277 //
1278 // These 3 tests can be done by evaluating the following
1279 // expression: ((mark - rsp) & (zero_bits - os::vm_page_size())),
1280 // assuming both stack pointer and pagesize have their
1281 // least significant bits clear.
1282 // NOTE: the mark is in swap_reg %rax as the result of cmpxchg
1283 subptr(swap_reg, rsp);
1284 andptr(swap_reg, zero_bits - os::vm_page_size());
1285
1286 // Save the test result, for recursive case, the result is zero
1287 movptr(Address(lock_reg, mark_offset), swap_reg);
1288
1289 if (PrintBiasedLockingStatistics) {
1290 cond_inc32(Assembler::zero,
1291 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
1292 }
1293 jcc(Assembler::zero, done);
1294
1295 bind(slow_case);
1296
1297 // Call the runtime routine for slow case
1298 call_VM(noreg,
1299 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
1300 lock_reg);
1301
1302 bind(done);
1303 }
1304 }
1305
1306
1307 // Unlocks an object. Used in monitorexit bytecode and
1308 // remove_activation. Throws an IllegalMonitorException if object is
1309 // not locked by current thread.
1310 //
1311 // Args:
1312 // rdx, c_rarg1: BasicObjectLock for lock
1313 //
1314 // Kills:
1315 // rax
1316 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
1317 // rscratch1 (scratch reg)
1318 // rax, rbx, rcx, rdx
1319 void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
1320 assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
1321 "The argument is only for looks. It must be c_rarg1");
1322
1323 if (UseHeavyMonitors) {
1324 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1325 } else {
1326 Label done;
1327
1328 const Register swap_reg = rax; // Must use rax for cmpxchg instruction
1329 const Register header_reg = LP64_ONLY(c_rarg2) NOT_LP64(rbx); // Will contain the old oopMark
1330 const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
1331
1332 save_bcp(); // Save in case of exception
1333
1334 // Convert from BasicObjectLock structure to object and BasicLock
1335 // structure Store the BasicLock address into %rax
1336 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
1337
1338 // Load oop into obj_reg(%c_rarg3)
1339 movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
1340
1341 // Free entry
1342 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), (int32_t)NULL_WORD);
1343
1344 if (UseBiasedLocking) {
1345 biased_locking_exit(obj_reg, header_reg, done);
1346 }
1347
1348 // Load the old header from BasicLock structure
1349 movptr(header_reg, Address(swap_reg,
1350 BasicLock::displaced_header_offset_in_bytes()));
1351
1352 // Test for recursion
1353 testptr(header_reg, header_reg);
1354
1355 // zero for recursive case
1356 jcc(Assembler::zero, done);
1357
1358 // Atomic swap back the old header
1359 lock();
1360 cmpxchgptr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1361
1362 // zero for simple unlock of a stack-lock case
1363 jcc(Assembler::zero, done);
1364
1365
1366 // Call the runtime routine for slow case.
1367 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), obj_reg); // restore obj
1368 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1369
1370 bind(done);
1371
1372 restore_bcp();
1373 }
1374 }
1375
1376 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
1377 Label& zero_continue) {
1378 assert(ProfileInterpreter, "must be profiling interpreter");
1379 movptr(mdp, Address(rbp, frame::interpreter_frame_mdp_offset * wordSize));
1380 testptr(mdp, mdp);
1381 jcc(Assembler::zero, zero_continue);
1382 }
1383
1384
1385 // Set the method data pointer for the current bcp.
|
40 #include "runtime/safepointMechanism.hpp"
41 #include "runtime/sharedRuntime.hpp"
42 #include "runtime/thread.inline.hpp"
43 #include "utilities/powerOfTwo.hpp"
44
45 // Implementation of InterpreterMacroAssembler
46
47 void InterpreterMacroAssembler::jump_to_entry(address entry) {
48 assert(entry, "Entry must have been generated by now");
49 jump(RuntimeAddress(entry));
50 }
51
52 void InterpreterMacroAssembler::profile_obj_type(Register obj, const Address& mdo_addr) {
53 Label update, next, none;
54
55 interp_verify_oop(obj, atos);
56
57 testptr(obj, obj);
58 jccb(Assembler::notZero, update);
59 orptr(mdo_addr, TypeEntries::null_seen);
60 jmp(next);
61
62 bind(update);
63 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
64 load_klass(obj, obj, tmp_load_klass);
65
66 xorptr(obj, mdo_addr);
67 testptr(obj, TypeEntries::type_klass_mask);
68 jccb(Assembler::zero, next); // klass seen before, nothing to
69 // do. The unknown bit may have been
70 // set already but no need to check.
71
72 testptr(obj, TypeEntries::type_unknown);
73 jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
74
75 cmpptr(mdo_addr, 0);
76 jccb(Assembler::equal, none);
77 cmpptr(mdo_addr, TypeEntries::null_seen);
78 jccb(Assembler::equal, none);
79 // There is a chance that the checks above (re-reading profiling
80 // data from memory) fail if another thread has just set the
1180 call_VM(noreg, CAST_FROM_FN_PTR(address,
1181 InterpreterRuntime::build_method_counters), method);
1182 movptr(mcs, Address(method,Method::method_counters_offset()));
1183 testptr(mcs, mcs);
1184 jcc(Assembler::zero, skip); // No MethodCounters allocated, OutOfMemory
1185 bind(has_counters);
1186 }
1187
1188
1189 // Lock object
1190 //
1191 // Args:
1192 // rdx, c_rarg1: BasicObjectLock to be used for locking
1193 //
1194 // Kills:
1195 // rax, rbx
1196 void InterpreterMacroAssembler::lock_object(Register lock_reg) {
1197 assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
1198 "The argument is only for looks. It must be c_rarg1");
1199
1200 if (LockingMode == LM_MONITOR) {
1201 call_VM(noreg,
1202 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
1203 lock_reg);
1204 } else {
1205 Label done;
1206
1207 const Register swap_reg = rax; // Must use rax for cmpxchg instruction
1208 const Register tmp_reg = rbx; // Will be passed to biased_locking_enter to avoid a
1209 // problematic case where tmp_reg = no_reg.
1210 const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
1211 const Register rklass_decode_tmp = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1212
1213 const int obj_offset = BasicObjectLock::obj_offset_in_bytes();
1214 const int lock_offset = BasicObjectLock::lock_offset_in_bytes ();
1215 const int mark_offset = lock_offset +
1216 BasicLock::displaced_header_offset_in_bytes();
1217
1218 Label slow_case;
1219
1220 // Load object pointer into obj_reg
1221 movptr(obj_reg, Address(lock_reg, obj_offset));
1222
1223 if (DiagnoseSyncOnValueBasedClasses != 0) {
1224 load_klass(tmp_reg, obj_reg, rklass_decode_tmp);
1225 movl(tmp_reg, Address(tmp_reg, Klass::access_flags_offset()));
1226 testl(tmp_reg, JVM_ACC_IS_VALUE_BASED_CLASS);
1227 jcc(Assembler::notZero, slow_case);
1228 }
1229
1230 if (UseBiasedLocking) {
1231 biased_locking_enter(lock_reg, obj_reg, swap_reg, tmp_reg, rklass_decode_tmp, false, done, &slow_case);
1232 }
1233
1234 if (LockingMode == LM_LIGHTWEIGHT) {
1235 #ifdef _LP64
1236 const Register thread = r15_thread;
1237 #else
1238 const Register thread = lock_reg;
1239 get_thread(thread);
1240 #endif
1241 // Load object header, prepare for CAS from unlocked to locked.
1242 movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1243 fast_lock_impl(obj_reg, swap_reg, thread, tmp_reg, slow_case);
1244 jmp(done);
1245 } else {
1246 // Load immediate 1 into swap_reg %rax
1247 movl(swap_reg, (int32_t)1);
1248
1249 // Load (object->mark() | 1) into swap_reg %rax
1250 orptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1251
1252 // Save (object->mark() | 1) into BasicLock's displaced header
1253 movptr(Address(lock_reg, mark_offset), swap_reg);
1254
1255 assert(lock_offset == 0,
1256 "displaced header must be first word in BasicObjectLock");
1257
1258 lock();
1259 cmpxchgptr(lock_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1260 if (PrintBiasedLockingStatistics) {
1261 cond_inc32(Assembler::zero,
1262 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
1263 }
1264 jcc(Assembler::zero, done);
1265
1266 const int zero_bits = LP64_ONLY(7) NOT_LP64(3);
1267
1268 // Fast check for recursive lock.
1269 //
1270 // Can apply the optimization only if this is a stack lock
1271 // allocated in this thread. For efficiency, we can focus on
1272 // recently allocated stack locks (instead of reading the stack
1273 // base and checking whether 'mark' points inside the current
1274 // thread stack):
1275 // 1) (mark & zero_bits) == 0, and
1276 // 2) rsp <= mark < mark + os::pagesize()
1277 //
1278 // Warning: rsp + os::pagesize can overflow the stack base. We must
1279 // neither apply the optimization for an inflated lock allocated
1280 // just above the thread stack (this is why condition 1 matters)
1281 // nor apply the optimization if the stack lock is inside the stack
1282 // of another thread. The latter is avoided even in case of overflow
1283 // because we have guard pages at the end of all stacks. Hence, if
1284 // we go over the stack base and hit the stack of another thread,
1285 // this should not be in a writeable area that could contain a
1286 // stack lock allocated by that thread. As a consequence, a stack
1287 // lock less than page size away from rsp is guaranteed to be
1288 // owned by the current thread.
1289 //
1290 // These 3 tests can be done by evaluating the following
1291 // expression: ((mark - rsp) & (zero_bits - os::vm_page_size())),
1292 // assuming both stack pointer and pagesize have their
1293 // least significant bits clear.
1294 // NOTE: the mark is in swap_reg %rax as the result of cmpxchg
1295 subptr(swap_reg, rsp);
1296 andptr(swap_reg, zero_bits - os::vm_page_size());
1297
1298 // Save the test result, for recursive case, the result is zero
1299 movptr(Address(lock_reg, mark_offset), swap_reg);
1300
1301 if (PrintBiasedLockingStatistics) {
1302 cond_inc32(Assembler::zero,
1303 ExternalAddress((address) BiasedLocking::fast_path_entry_count_addr()));
1304 }
1305 jcc(Assembler::zero, done);
1306 }
1307 bind(slow_case);
1308
1309 // Call the runtime routine for slow case
1310 if (LockingMode == LM_LIGHTWEIGHT) {
1311 call_VM(noreg,
1312 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter_obj),
1313 obj_reg);
1314 } else {
1315 call_VM(noreg,
1316 CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorenter),
1317 lock_reg);
1318 }
1319 bind(done);
1320 }
1321 }
1322
1323
1324 // Unlocks an object. Used in monitorexit bytecode and
1325 // remove_activation. Throws an IllegalMonitorException if object is
1326 // not locked by current thread.
1327 //
1328 // Args:
1329 // rdx, c_rarg1: BasicObjectLock for lock
1330 //
1331 // Kills:
1332 // rax
1333 // c_rarg0, c_rarg1, c_rarg2, c_rarg3, ... (param regs)
1334 // rscratch1 (scratch reg)
1335 // rax, rbx, rcx, rdx
1336 void InterpreterMacroAssembler::unlock_object(Register lock_reg) {
1337 assert(lock_reg == LP64_ONLY(c_rarg1) NOT_LP64(rdx),
1338 "The argument is only for looks. It must be c_rarg1");
1339
1340 if (LockingMode == LM_MONITOR) {
1341 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1342 } else {
1343 Label done, slow_case;
1344
1345 const Register swap_reg = rax; // Must use rax for cmpxchg instruction
1346 const Register header_reg = LP64_ONLY(c_rarg2) NOT_LP64(rbx); // Will contain the old oopMark
1347 const Register obj_reg = LP64_ONLY(c_rarg3) NOT_LP64(rcx); // Will contain the oop
1348
1349 save_bcp(); // Save in case of exception
1350
1351 if (LockingMode != LM_LIGHTWEIGHT) {
1352 // Convert from BasicObjectLock structure to object and BasicLock
1353 // structure Store the BasicLock address into %rax
1354 lea(swap_reg, Address(lock_reg, BasicObjectLock::lock_offset_in_bytes()));
1355 }
1356
1357 // Load oop into obj_reg(%c_rarg3)
1358 movptr(obj_reg, Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()));
1359
1360 // Free entry
1361 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), (int32_t)NULL_WORD);
1362
1363 if (LockingMode == LM_LIGHTWEIGHT) {
1364 #ifdef _LP64
1365 const Register thread = r15_thread;
1366 #else
1367 const Register thread = header_reg;
1368 get_thread(thread);
1369 #endif
1370 // Handle unstructured locking.
1371 Register tmp = swap_reg;
1372 movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
1373 cmpptr(obj_reg, Address(thread, tmp, Address::times_1, -oopSize));
1374 jcc(Assembler::notEqual, slow_case);
1375 // Try to swing header from locked to unlock.
1376 movptr(swap_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1377 andptr(swap_reg, ~(int32_t)markWord::lock_mask_in_place);
1378 fast_unlock_impl(obj_reg, swap_reg, header_reg, slow_case);
1379 jmp(done);
1380 } else {
1381 if (UseBiasedLocking) {
1382 biased_locking_exit(obj_reg, header_reg, done);
1383 }
1384
1385 // Load the old header from BasicLock structure
1386 movptr(header_reg, Address(swap_reg,
1387 BasicLock::displaced_header_offset_in_bytes()));
1388
1389 // Test for recursion
1390 testptr(header_reg, header_reg);
1391
1392 // zero for recursive case
1393 jcc(Assembler::zero, done);
1394
1395 // Atomic swap back the old header
1396 lock();
1397 cmpxchgptr(header_reg, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1398
1399 // zero for simple unlock of a stack-lock case
1400 jcc(Assembler::zero, done);
1401 }
1402
1403 bind(slow_case);
1404 // Call the runtime routine for slow case.
1405 movptr(Address(lock_reg, BasicObjectLock::obj_offset_in_bytes()), obj_reg); // restore obj
1406 call_VM_leaf(CAST_FROM_FN_PTR(address, InterpreterRuntime::monitorexit), lock_reg);
1407
1408 bind(done);
1409
1410 restore_bcp();
1411 }
1412 }
1413
1414 void InterpreterMacroAssembler::test_method_data_pointer(Register mdp,
1415 Label& zero_continue) {
1416 assert(ProfileInterpreter, "must be profiling interpreter");
1417 movptr(mdp, Address(rbp, frame::interpreter_frame_mdp_offset * wordSize));
1418 testptr(mdp, mdp);
1419 jcc(Assembler::zero, zero_continue);
1420 }
1421
1422
1423 // Set the method data pointer for the current bcp.
|