207 ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
208 ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
209 orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
210 cmp(rscratch1, zr); // Sets flags for result
211 cbnz(rscratch1, cont);
212 // need a release store here
213 lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
214 stlr(zr, tmp); // set unowned
215
216 bind(cont);
217 // flag == EQ indicates success
218 // flag == NE indicates failure
219 br(Assembler::NE, no_count);
220
221 bind(count);
222 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
223
224 bind(no_count);
225 }
226
227 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register t1,
228 Register t2, Register t3) {
229 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
230 assert_different_registers(obj, t1, t2, t3);
231
232 // Handle inflated monitor.
233 Label inflated;
234 // Finish fast lock successfully. MUST branch to with flag == EQ
235 Label locked;
236 // Finish fast lock unsuccessfully. MUST branch to with flag == NE
237 Label slow_path;
238
239 if (DiagnoseSyncOnValueBasedClasses != 0) {
240 load_klass(t1, obj);
241 ldrw(t1, Address(t1, Klass::access_flags_offset()));
242 tstw(t1, JVM_ACC_IS_VALUE_BASED_CLASS);
243 br(Assembler::NE, slow_path);
244 }
245
246 const Register t1_mark = t1;
247
248 { // Lightweight locking
249
250 // Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
251 Label push;
252
253 const Register t2_top = t2;
254 const Register t3_t = t3;
255
256 // Check if lock-stack is full.
257 ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
258 cmpw(t2_top, (unsigned)LockStack::end_offset() - 1);
259 br(Assembler::GT, slow_path);
260
261 // Check if recursive.
262 subw(t3_t, t2_top, oopSize);
263 ldr(t3_t, Address(rthread, t3_t));
264 cmp(obj, t3_t);
265 br(Assembler::EQ, push);
266
267 // Relaxed normal load to check for monitor. Optimization for monitor case.
268 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
269 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
270
271 // Not inflated
272 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
273
274 // Try to lock. Transition lock-bits 0b01 => 0b00
275 orr(t1_mark, t1_mark, markWord::unlocked_value);
276 eor(t3_t, t1_mark, markWord::unlocked_value);
277 cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
278 /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
279 br(Assembler::NE, slow_path);
280
281 bind(push);
282 // After successful lock, push object on lock-stack.
283 str(obj, Address(rthread, t2_top));
284 addw(t2_top, t2_top, oopSize);
285 strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
286 b(locked);
287 }
288
289 { // Handle inflated monitor.
290 bind(inflated);
291
292 // mark contains the tagged ObjectMonitor*.
293 const Register t1_tagged_monitor = t1_mark;
294 const uintptr_t monitor_tag = markWord::monitor_value;
295 const Register t2_owner_addr = t2;
296 const Register t3_owner = t3;
297
298 // Compute owner address.
299 lea(t2_owner_addr, Address(t1_tagged_monitor, (in_bytes(ObjectMonitor::owner_offset()) - monitor_tag)));
300
301 // CAS owner (null => current thread).
302 cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
303 /*release*/ false, /*weak*/ false, t3_owner);
304 br(Assembler::EQ, locked);
305
306 // Check if recursive.
307 cmp(t3_owner, rthread);
308 br(Assembler::NE, slow_path);
309
310 // Recursive.
311 increment(Address(t1_tagged_monitor, in_bytes(ObjectMonitor::recursions_offset()) - monitor_tag), 1);
312 }
313
314 bind(locked);
315 increment(Address(rthread, JavaThread::held_monitor_count_offset()));
316
317 #ifdef ASSERT
318 // Check that locked label is reached with Flags == EQ.
319 Label flag_correct;
320 br(Assembler::EQ, flag_correct);
321 stop("Fast Lock Flag != EQ");
322 #endif
323
324 bind(slow_path);
325 #ifdef ASSERT
326 // Check that slow_path label is reached with Flags == NE.
327 br(Assembler::NE, flag_correct);
328 stop("Fast Lock Flag != NE");
329 bind(flag_correct);
330 #endif
331 // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
332 }
333
334 void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register t1, Register t2,
335 Register t3) {
336 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
337 assert_different_registers(obj, t1, t2, t3);
338
339 // Handle inflated monitor.
340 Label inflated, inflated_load_monitor;
341 // Finish fast unlock successfully. MUST branch to with flag == EQ
342 Label unlocked;
343 // Finish fast unlock unsuccessfully. MUST branch to with flag == NE
344 Label slow_path;
345
346 const Register t1_mark = t1;
347 const Register t2_top = t2;
348 const Register t3_t = t3;
349
350 { // Lightweight unlock
351
352 // Check if obj is top of lock-stack.
353 ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
354 subw(t2_top, t2_top, oopSize);
355 ldr(t3_t, Address(rthread, t2_top));
356 cmp(obj, t3_t);
357 // Top of lock stack was not obj. Must be monitor.
358 br(Assembler::NE, inflated_load_monitor);
359
360 // Pop lock-stack.
361 DEBUG_ONLY(str(zr, Address(rthread, t2_top));)
362 strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
363
364 // Check if recursive.
365 subw(t3_t, t2_top, oopSize);
366 ldr(t3_t, Address(rthread, t3_t));
367 cmp(obj, t3_t);
368 br(Assembler::EQ, unlocked);
369
370 // Not recursive.
371 // Load Mark.
372 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
373
374 // Check header for monitor (0b10).
375 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
376
377 // Try to unlock. Transition lock bits 0b00 => 0b01
378 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
379 orr(t3_t, t1_mark, markWord::unlocked_value);
380 cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
381 /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
382 br(Assembler::EQ, unlocked);
383
384 // Compare and exchange failed.
385 // Restore lock-stack and handle the unlock in runtime.
386 DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
387 addw(t2_top, t2_top, oopSize);
388 str(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
389 b(slow_path);
390 }
391
392
393 { // Handle inflated monitor.
394 bind(inflated_load_monitor);
395 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
396 #ifdef ASSERT
397 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
398 stop("Fast Unlock not monitor");
399 #endif
400
401 bind(inflated);
402
403 #ifdef ASSERT
404 Label check_done;
405 subw(t2_top, t2_top, oopSize);
406 cmpw(t2_top, in_bytes(JavaThread::lock_stack_base_offset()));
407 br(Assembler::LT, check_done);
408 ldr(t3_t, Address(rthread, t2_top));
409 cmp(obj, t3_t);
410 br(Assembler::NE, inflated);
411 stop("Fast Unlock lock on stack");
412 bind(check_done);
413 #endif
414
415 // mark contains the tagged ObjectMonitor*.
416 const Register t1_monitor = t1_mark;
417 const uintptr_t monitor_tag = markWord::monitor_value;
418
419 // Untag the monitor.
420 sub(t1_monitor, t1_mark, monitor_tag);
421
422 const Register t2_recursions = t2;
423 Label not_recursive;
424
425 // Check if recursive.
426 ldr(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
427 cbz(t2_recursions, not_recursive);
428
429 // Recursive unlock.
430 sub(t2_recursions, t2_recursions, 1u);
431 str(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
432 // Set flag == EQ
433 cmp(t2_recursions, t2_recursions);
434 b(unlocked);
435
436 bind(not_recursive);
437
438 Label release;
439 const Register t2_owner_addr = t2;
440
441 // Compute owner address.
442 lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
443
444 // Check if the entry lists are empty.
445 ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
446 ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
447 orr(rscratch1, rscratch1, t3_t);
448 cmp(rscratch1, zr);
449 br(Assembler::EQ, release);
450
451 // The owner may be anonymous and we removed the last obj entry in
452 // the lock-stack. This loses the information about the owner.
453 // Write the thread to the owner field so the runtime knows the owner.
454 str(rthread, Address(t2_owner_addr));
455 b(slow_path);
456
457 bind(release);
458 // Set owner to null.
459 // Release to satisfy the JMM
460 stlr(zr, t2_owner_addr);
461 }
462
463 bind(unlocked);
464 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
465
466 #ifdef ASSERT
467 // Check that unlocked label is reached with Flags == EQ.
468 Label flag_correct;
469 br(Assembler::EQ, flag_correct);
470 stop("Fast Unlock Flag != EQ");
471 #endif
472
473 bind(slow_path);
474 #ifdef ASSERT
475 // Check that slow_path label is reached with Flags == NE.
476 br(Assembler::NE, flag_correct);
477 stop("Fast Unlock Flag != NE");
478 bind(flag_correct);
479 #endif
480 // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
2477 sve_and(vtmp, T, min_jlong);
2478 sve_orr(vtmp, T, jlong_cast(1.0));
2479 break;
2480 default:
2481 assert(false, "unsupported");
2482 ShouldNotReachHere();
2483 }
2484 sve_sel(dst, T, pgtmp, vtmp, src); // Select either from src or vtmp based on the predicate register pgtmp
2485 // Result in dst
2486 }
2487
2488 bool C2_MacroAssembler::in_scratch_emit_size() {
2489 if (ciEnv::current()->task() != nullptr) {
2490 PhaseOutput* phase_output = Compile::current()->output();
2491 if (phase_output != nullptr && phase_output->in_scratch_emit_size()) {
2492 return true;
2493 }
2494 }
2495 return MacroAssembler::in_scratch_emit_size();
2496 }
|
207 ldr(rscratch1, Address(tmp, ObjectMonitor::EntryList_offset()));
208 ldr(disp_hdr, Address(tmp, ObjectMonitor::cxq_offset()));
209 orr(rscratch1, rscratch1, disp_hdr); // Will be 0 if both are 0.
210 cmp(rscratch1, zr); // Sets flags for result
211 cbnz(rscratch1, cont);
212 // need a release store here
213 lea(tmp, Address(tmp, ObjectMonitor::owner_offset()));
214 stlr(zr, tmp); // set unowned
215
216 bind(cont);
217 // flag == EQ indicates success
218 // flag == NE indicates failure
219 br(Assembler::NE, no_count);
220
221 bind(count);
222 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
223
224 bind(no_count);
225 }
226
227 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register t1,
228 Register t2, Register t3) {
229 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
230 assert_different_registers(obj, box, t1, t2, t3);
231
232 // Handle inflated monitor.
233 Label inflated;
234 // Finish fast lock successfully. MUST branch to with flag == EQ
235 Label locked;
236 // Finish fast lock unsuccessfully. MUST branch to with flag == NE
237 Label slow_path;
238
239 // Clear box. TODO[OMWorld]: Is this necessary? May also defer this to not write twice.
240 str(zr, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
241
242 if (DiagnoseSyncOnValueBasedClasses != 0) {
243 load_klass(t1, obj);
244 ldrw(t1, Address(t1, Klass::access_flags_offset()));
245 tstw(t1, JVM_ACC_IS_VALUE_BASED_CLASS);
246 br(Assembler::NE, slow_path);
247 }
248
249 const Register t1_mark = t1;
250 const Register t3_t = t3;
251
252 { // Lightweight locking
253
254 // Push lock to the lock stack and finish successfully. MUST branch to with flag == EQ
255 Label push;
256
257 const Register t2_top = t2;
258
259 // Check if lock-stack is full.
260 ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
261 cmpw(t2_top, (unsigned)LockStack::end_offset() - 1);
262 br(Assembler::GT, slow_path);
263
264 // Check if recursive.
265 subw(t3_t, t2_top, oopSize);
266 ldr(t3_t, Address(rthread, t3_t));
267 cmp(obj, t3_t);
268 br(Assembler::EQ, push);
269
270 // Relaxed normal load to check for monitor. Optimization for monitor case.
271 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
272 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
273
274 // Not inflated
275 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a lea");
276
277 // Try to lock. Transition lock-bits 0b01 => 0b00
278 orr(t1_mark, t1_mark, markWord::unlocked_value);
279 eor(t3_t, t1_mark, markWord::unlocked_value);
280 cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
281 /*acquire*/ true, /*release*/ false, /*weak*/ false, noreg);
282 br(Assembler::NE, slow_path);
283
284 bind(push);
285 // After successful lock, push object on lock-stack.
286 str(obj, Address(rthread, t2_top));
287 addw(t2_top, t2_top, oopSize);
288 strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
289 b(locked);
290 }
291
292 { // Handle inflated monitor.
293 bind(inflated);
294
295 if (!OMUseC2Cache) {
296 // Set Flags == NE
297 cmp(zr, obj);
298 b(slow_path);
299 } else {
300
301 if (OMCacheHitRate) increment(Address(rthread, JavaThread::lock_lookup_offset()));
302
303 Label monitor_found;
304
305 // Load cache address
306 lea(t3_t, Address(rthread, JavaThread::om_cache_oops_offset()));
307
308 const int num_unrolled = MIN2(OMC2UnrollCacheEntries, OMCacheSize);
309 for (int i = 0; i < num_unrolled; i++) {
310 ldr(t1, Address(t3_t));
311 cmp(obj, t1);
312 br(Assembler::EQ, monitor_found);
313 if (i + 1 != num_unrolled) {
314 increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
315 }
316 }
317
318 if (num_unrolled == 0 || (OMC2UnrollCacheLookupLoopTail && num_unrolled != OMCacheSize)) {
319 if (num_unrolled != 0) {
320 // Loop after unrolling, advance iterator.
321 increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
322 }
323
324 Label loop;
325
326 // Search for obj in cache.
327 bind(loop);
328
329 // Check for match.
330 ldr(t1, Address(t3_t));
331 cmp(obj, t1);
332 br(Assembler::EQ, monitor_found);
333
334 // Search until null encountered, guaranteed _null_sentinel at end.
335 increment(t3_t, in_bytes(OMCache::oop_to_oop_difference()));
336 cbnz(t1, loop);
337 // Cache Miss, NE set from cmp above, cbnz does not set flags
338 b(slow_path);
339 } else {
340 b(slow_path);
341 }
342
343 bind(monitor_found);
344 ldr(t1, Address(t3_t, OMCache::oop_to_monitor_difference()));
345 if (OMCacheHitRate) increment(Address(rthread, JavaThread::lock_hit_offset()));
346
347 // ObjectMonitor* is in t1
348 const Register t1_monitor = t1;
349 const Register t2_owner_addr = t2;
350 const Register t3_owner = t3;
351
352 Label recursive;
353 Label monitor_locked;
354
355 // Compute owner address.
356 lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
357
358 if (OMRecursiveFastPath) {
359 ldr(t3_owner, Address(t2_owner_addr));
360 cmp(t3_owner, rthread);
361 br(Assembler::EQ, recursive);
362 }
363
364 // CAS owner (null => current thread).
365 cmpxchg(t2_owner_addr, zr, rthread, Assembler::xword, /*acquire*/ true,
366 /*release*/ false, /*weak*/ false, t3_owner);
367 br(Assembler::EQ, monitor_locked);
368
369 if (OMRecursiveFastPath) {
370 b(slow_path);
371 } else {
372 // Check if recursive.
373 cmp(t3_owner, rthread);
374 br(Assembler::NE, slow_path);
375 }
376
377 // Recursive.
378 bind(recursive);
379 increment(Address(t1_monitor, ObjectMonitor::recursions_offset()), 1);
380
381 bind(monitor_locked);
382 str(t1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
383 }
384
385 }
386
387 bind(locked);
388 increment(Address(rthread, JavaThread::held_monitor_count_offset()));
389
390 #ifdef ASSERT
391 // Check that locked label is reached with Flags == EQ.
392 Label flag_correct;
393 br(Assembler::EQ, flag_correct);
394 stop("Fast Lock Flag != EQ");
395 #endif
396
397 bind(slow_path);
398 #ifdef ASSERT
399 // Check that slow_path label is reached with Flags == NE.
400 br(Assembler::NE, flag_correct);
401 stop("Fast Lock Flag != NE");
402 bind(flag_correct);
403 #endif
404 // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
405 }
406
407 void C2_MacroAssembler::fast_unlock_lightweight(Register obj, Register box, Register t1,
408 Register t2, Register t3) {
409 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
410 assert_different_registers(obj, box, t1, t2, t3);
411
412 // Handle inflated monitor.
413 Label inflated, inflated_load_monitor;
414 // Finish fast unlock successfully. MUST branch to with flag == EQ
415 Label unlocked;
416 // Finish fast unlock unsuccessfully. MUST branch to with flag == NE
417 Label slow_path;
418
419 const Register t1_mark = t1;
420 const Register t2_top = t2;
421 const Register t3_t = t3;
422
423 { // Lightweight unlock
424
425 Label push_and_slow_path;
426
427 // Check if obj is top of lock-stack.
428 ldrw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
429 subw(t2_top, t2_top, oopSize);
430 ldr(t3_t, Address(rthread, t2_top));
431 cmp(obj, t3_t);
432 // Top of lock stack was not obj. Must be monitor.
433 br(Assembler::NE, inflated_load_monitor);
434
435 // Pop lock-stack.
436 DEBUG_ONLY(str(zr, Address(rthread, t2_top));)
437 strw(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
438
439 // Check if recursive.
440 subw(t3_t, t2_top, oopSize);
441 ldr(t3_t, Address(rthread, t3_t));
442 cmp(obj, t3_t);
443 br(Assembler::EQ, unlocked);
444
445 // Not recursive.
446 // Load Mark.
447 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
448
449 // Check header for monitor (0b10).
450 // Because we got here by popping (meaning we pushed in locked)
451 // there will be no monitor in the box. So we need to push back the obj
452 // so that the runtime can fix any potential anonymous owner.
453 tbnz(t1_mark, exact_log2(markWord::monitor_value), push_and_slow_path);
454
455 // Try to unlock. Transition lock bits 0b00 => 0b01
456 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea");
457 orr(t3_t, t1_mark, markWord::unlocked_value);
458 cmpxchg(/*addr*/ obj, /*expected*/ t1_mark, /*new*/ t3_t, Assembler::xword,
459 /*acquire*/ false, /*release*/ true, /*weak*/ false, noreg);
460 br(Assembler::EQ, unlocked);
461
462 bind(push_and_slow_path);
463 // Compare and exchange failed.
464 // Restore lock-stack and handle the unlock in runtime.
465 DEBUG_ONLY(str(obj, Address(rthread, t2_top));)
466 addw(t2_top, t2_top, oopSize);
467 str(t2_top, Address(rthread, JavaThread::lock_stack_top_offset()));
468 b(slow_path);
469 }
470
471
472 { // Handle inflated monitor.
473 bind(inflated_load_monitor);
474 ldr(t1_mark, Address(obj, oopDesc::mark_offset_in_bytes()));
475 #ifdef ASSERT
476 tbnz(t1_mark, exact_log2(markWord::monitor_value), inflated);
477 stop("Fast Unlock not monitor");
478 #endif
479
480 bind(inflated);
481
482 #ifdef ASSERT
483 Label check_done;
484 subw(t2_top, t2_top, oopSize);
485 cmpw(t2_top, in_bytes(JavaThread::lock_stack_base_offset()));
486 br(Assembler::LT, check_done);
487 ldr(t3_t, Address(rthread, t2_top));
488 cmp(obj, t3_t);
489 br(Assembler::NE, inflated);
490 stop("Fast Unlock lock on stack");
491 bind(check_done);
492 #endif
493
494 if (!OMUseC2Cache) {
495 b(slow_path);
496 } else {
497 const Register t1_monitor = t1;
498
499 if (OMCacheHitRate) increment(Address(rthread, JavaThread::unlock_lookup_offset()));
500 ldr(t1_monitor, Address(box, BasicLock::object_monitor_cache_offset_in_bytes()));
501 // TODO: Cleanup these constants (with an enum and asserts)
502 cmp(t1_monitor, (uint8_t)2);
503 // Non symmetrical, take slow path monitor == 0 or 1, 0 and 1 < 2, both LS and NE
504 br(Assembler::LO, slow_path);
505 if (OMCacheHitRate) increment(Address(rthread, JavaThread::unlock_hit_offset()));
506
507 const Register t2_recursions = t2;
508 Label not_recursive;
509
510 // Check if recursive.
511 ldr(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
512 cbz(t2_recursions, not_recursive);
513
514 // Recursive unlock.
515 sub(t2_recursions, t2_recursions, 1u);
516 str(t2_recursions, Address(t1_monitor, ObjectMonitor::recursions_offset()));
517 // Set flag == EQ
518 cmp(t2_recursions, t2_recursions);
519 b(unlocked);
520
521 bind(not_recursive);
522
523 Label release;
524 const Register t2_owner_addr = t2;
525
526 // Compute owner address.
527 lea(t2_owner_addr, Address(t1_monitor, ObjectMonitor::owner_offset()));
528
529 // Check if the entry lists are empty.
530 ldr(rscratch1, Address(t1_monitor, ObjectMonitor::EntryList_offset()));
531 ldr(t3_t, Address(t1_monitor, ObjectMonitor::cxq_offset()));
532 orr(rscratch1, rscratch1, t3_t);
533 cmp(rscratch1, zr);
534 br(Assembler::EQ, release);
535
536 // The owner may be anonymous and we removed the last obj entry in
537 // the lock-stack. This loses the information about the owner.
538 // Write the thread to the owner field so the runtime knows the owner.
539 str(rthread, Address(t2_owner_addr));
540 b(slow_path);
541
542 bind(release);
543 // Set owner to null.
544 // Release to satisfy the JMM
545 stlr(zr, t2_owner_addr);
546 }
547 }
548
549 bind(unlocked);
550 decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
551
552 #ifdef ASSERT
553 // Check that unlocked label is reached with Flags == EQ.
554 Label flag_correct;
555 br(Assembler::EQ, flag_correct);
556 stop("Fast Unlock Flag != EQ");
557 #endif
558
559 bind(slow_path);
560 #ifdef ASSERT
561 // Check that slow_path label is reached with Flags == NE.
562 br(Assembler::NE, flag_correct);
563 stop("Fast Unlock Flag != NE");
564 bind(flag_correct);
565 #endif
566 // C2 uses the value of Flags (NE vs EQ) to determine the continuation.
2563 sve_and(vtmp, T, min_jlong);
2564 sve_orr(vtmp, T, jlong_cast(1.0));
2565 break;
2566 default:
2567 assert(false, "unsupported");
2568 ShouldNotReachHere();
2569 }
2570 sve_sel(dst, T, pgtmp, vtmp, src); // Select either from src or vtmp based on the predicate register pgtmp
2571 // Result in dst
2572 }
2573
2574 bool C2_MacroAssembler::in_scratch_emit_size() {
2575 if (ciEnv::current()->task() != nullptr) {
2576 PhaseOutput* phase_output = Compile::current()->output();
2577 if (phase_output != nullptr && phase_output->in_scratch_emit_size()) {
2578 return true;
2579 }
2580 }
2581 return MacroAssembler::in_scratch_emit_size();
2582 }
2583
2584 void C2_MacroAssembler::load_nklass_compact(Register dst, Register obj, Register index, int scale, int disp) {
2585 // Note: Don't clobber obj anywhere in that method!
2586
2587 // The incoming address is pointing into obj-start + klass_offset_in_bytes. We need to extract
2588 // obj-start, so that we can load from the object's mark-word instead. Usually the address
2589 // comes as obj-start in obj and klass_offset_in_bytes in disp. However, sometimes C2
2590 // emits code that pre-computes obj-start + klass_offset_in_bytes into a register, and
2591 // then passes that register as obj and 0 in disp. The following code extracts the base
2592 // and offset to load the mark-word.
2593 int offset = oopDesc::mark_offset_in_bytes() + disp - oopDesc::klass_offset_in_bytes();
2594 if (index == noreg) {
2595 ldr(dst, Address(obj, offset));
2596 } else {
2597 lea(dst, Address(obj, index, Address::lsl(scale)));
2598 ldr(dst, Address(dst, offset));
2599 }
2600 lsr(dst, dst, markWord::klass_shift);
2601 }
|