312
313 #ifndef _LP64
314 // The object is inflated.
315
316 // boxReg refers to the on-stack BasicLock in the current frame.
317 // We'd like to write:
318 // set box->_displaced_header = markWord::unused_mark(). Any non-0 value suffices.
319 // This is convenient but results a ST-before-CAS penalty. The following CAS suffers
320 // additional latency as we have another ST in the store buffer that must drain.
321
322 // avoid ST-before-CAS
323 // register juggle because we need tmpReg for cmpxchgptr below
324 movptr(scrReg, boxReg);
325 movptr(boxReg, tmpReg); // consider: LEA box, [tmp-2]
326
327 // Optimistic form: consider XORL tmpReg,tmpReg
328 movptr(tmpReg, NULL_WORD);
329
330 // Appears unlocked - try to swing _owner from null to non-null.
331 // Ideally, I'd manifest "Self" with get_thread and then attempt
332 // to CAS the register containing Self into m->Owner.
333 // But we don't have enough registers, so instead we can either try to CAS
334 // rsp or the address of the box (in scr) into &m->owner. If the CAS succeeds
335 // we later store "Self" into m->Owner. Transiently storing a stack address
336 // (rsp or the address of the box) into m->owner is harmless.
337 // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand.
338 lock();
339 cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
340 movptr(Address(scrReg, 0), 3); // box->_displaced_header = 3
341 // If we weren't able to swing _owner from null to the BasicLock
342 // then take the slow path.
343 jccb (Assembler::notZero, NO_COUNT);
344 // update _owner from BasicLock to thread
345 get_thread (scrReg); // beware: clobbers ICCs
346 movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
347 xorptr(boxReg, boxReg); // set icc.ZFlag = 1 to indicate success
348
349 // If the CAS fails we can either retry or pass control to the slow path.
350 // We use the latter tactic.
351 // Pass the CAS result in the icc.ZFlag into DONE_LABEL
352 // If the CAS was successful ...
353 // Self has acquired the lock
354 // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
355 // Intentional fall-through into DONE_LABEL ...
356 #else // _LP64
357 // It's inflated and we use scrReg for ObjectMonitor* in this section.
358 movq(scrReg, tmpReg);
359 xorq(tmpReg, tmpReg);
360 lock();
361 cmpxchgptr(thread, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
362 // Unconditionally set box->_displaced_header = markWord::unused_mark().
363 // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
364 movptr(Address(boxReg, 0), checked_cast<int32_t>(markWord::unused_mark().value()));
365 // Propagate ICC.ZF from CAS above into DONE_LABEL.
366 jccb(Assembler::equal, COUNT); // CAS above succeeded; propagate ZF = 1 (success)
367
368 cmpptr(thread, rax); // Check if we are already the owner (recursive lock)
369 jccb(Assembler::notEqual, NO_COUNT); // If not recursive, ZF = 0 at this point (fail)
370 incq(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
371 xorq(rax, rax); // Set ZF = 1 (success) for recursive lock, denoting locking success
372 #endif // _LP64
373 bind(DONE_LABEL);
374
375 // ZFlag == 1 count in fast path
376 // ZFlag == 0 count in slow path
377 jccb(Assembler::notZero, NO_COUNT); // jump if ZFlag == 0
378
379 bind(COUNT);
380 // Count monitors in fast path
381 increment(Address(thread, JavaThread::held_monitor_count_offset()));
382
383 xorl(tmpReg, tmpReg); // Set ZF == 1
384
385 bind(NO_COUNT);
386
387 // At NO_COUNT the icc ZFlag is set as follows ...
388 // fast_unlock uses the same protocol.
389 // ZFlag == 1 -> Success
390 // ZFlag == 0 -> Failure - force control through the slow path
391 }
392
393 // obj: object to unlock
394 // box: box address (displaced header location), killed. Must be EAX.
395 // tmp: killed, cannot be obj nor box.
396 //
397 // Some commentary on balanced locking:
398 //
399 // fast_lock and fast_unlock are emitted only for provably balanced lock sites.
400 // Methods that don't have provably balanced locking are forced to run in the
401 // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
402 // The interpreter provides two properties:
501
502 bind (LSuccess);
503 testl (boxReg, 0); // set ICC.ZF=1 to indicate success
504 jmpb (DONE_LABEL);
505
506 if (LockingMode == LM_LEGACY) {
507 bind (Stacked);
508 movptr(tmpReg, Address (boxReg, 0)); // re-fetch
509 lock();
510 cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
511 // Intentional fall-thru into DONE_LABEL
512 }
513
514 bind(DONE_LABEL);
515
516 // ZFlag == 1 count in fast path
517 // ZFlag == 0 count in slow path
518 jccb(Assembler::notZero, NO_COUNT);
519
520 bind(COUNT);
521 // Count monitors in fast path
522 #ifndef _LP64
523 get_thread(tmpReg);
524 decrementl(Address(tmpReg, JavaThread::held_monitor_count_offset()));
525 #else // _LP64
526 decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
527 #endif
528
529 xorl(tmpReg, tmpReg); // Set ZF == 1
530
531 bind(NO_COUNT);
532 }
533
534 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register rax_reg,
535 Register t, Register thread) {
536 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
537 assert(rax_reg == rax, "Used for CAS");
538 assert_different_registers(obj, box, rax_reg, t, thread);
539
540 // Handle inflated monitor.
541 Label inflated;
542 // Finish fast lock successfully. ZF value is irrelevant.
543 Label locked;
544 // Finish fast lock unsuccessfully. MUST jump with ZF == 0
545 Label slow_path;
546
547 if (UseObjectMonitorTable) {
630 cmpptr(obj, Address(t));
631 jccb(Assembler::equal, monitor_found);
632
633 // Search until null encountered, guaranteed _null_sentinel at end.
634 cmpptr(Address(t), 1);
635 jcc(Assembler::below, slow_path); // 0 check, but with ZF=0 when *t == 0
636 increment(t, in_bytes(OMCache::oop_to_oop_difference()));
637 jmpb(loop);
638
639 // Cache hit.
640 bind(monitor_found);
641 movptr(monitor, Address(t, OMCache::oop_to_monitor_difference()));
642 }
643 const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
644 const Address recursions_address(monitor, ObjectMonitor::recursions_offset() - monitor_tag);
645 const Address owner_address(monitor, ObjectMonitor::owner_offset() - monitor_tag);
646
647 Label monitor_locked;
648 // Lock the monitor.
649
650 // CAS owner (null => current thread).
651 xorptr(rax_reg, rax_reg);
652 lock(); cmpxchgptr(thread, owner_address);
653 jccb(Assembler::equal, monitor_locked);
654
655 // Check if recursive.
656 cmpptr(thread, rax_reg);
657 jccb(Assembler::notEqual, slow_path);
658
659 // Recursive.
660 increment(recursions_address);
661
662 bind(monitor_locked);
663 if (UseObjectMonitorTable) {
664 // Cache the monitor for unlock
665 movptr(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), monitor);
666 }
667 }
668
669 bind(locked);
670 increment(Address(thread, JavaThread::held_monitor_count_offset()));
671 // Set ZF = 1
672 xorl(rax_reg, rax_reg);
673
674 #ifdef ASSERT
675 // Check that locked label is reached with ZF set.
676 Label zf_correct;
677 Label zf_bad_zero;
678 jcc(Assembler::zero, zf_correct);
679 jmp(zf_bad_zero);
680 #endif
681
682 bind(slow_path);
683 #ifdef ASSERT
684 // Check that slow_path label is reached with ZF not set.
685 jcc(Assembler::notZero, zf_correct);
686 stop("Fast Lock ZF != 0");
687 bind(zf_bad_zero);
688 stop("Fast Lock ZF != 1");
689 bind(zf_correct);
690 #endif
814 // Check if there is a successor.
815 cmpptr(succ_address, NULL_WORD);
816 jccb(Assembler::notZero, unlocked); // If so we are done.
817
818 // Save the monitor pointer in the current thread, so we can try to
819 // reacquire the lock in SharedRuntime::monitor_exit_helper().
820 if (!UseObjectMonitorTable) {
821 andptr(monitor, ~(int32_t)markWord::monitor_value);
822 }
823 movptr(Address(thread, JavaThread::unlocked_inflated_monitor_offset()), monitor);
824
825 orl(t, 1); // Fast Unlock ZF = 0
826 jmpb(slow_path);
827
828 // Recursive unlock.
829 bind(recursive);
830 decrement(recursions_address);
831 }
832
833 bind(unlocked);
834 decrement(Address(thread, JavaThread::held_monitor_count_offset()));
835 xorl(t, t); // Fast Unlock ZF = 1
836
837 #ifdef ASSERT
838 // Check that unlocked label is reached with ZF set.
839 Label zf_correct;
840 jcc(Assembler::zero, zf_correct);
841 stop("Fast Unlock ZF != 1");
842 #endif
843
844 bind(slow_path);
845 if (stub != nullptr) {
846 bind(stub->slow_path_continuation());
847 }
848 #ifdef ASSERT
849 // Check that stub->continuation() label is reached with ZF not set.
850 jccb(Assembler::notZero, zf_correct);
851 stop("Fast Unlock ZF != 0");
852 bind(zf_correct);
853 #endif
854 // C2 uses the value of ZF to determine the continuation.
|
312
313 #ifndef _LP64
314 // The object is inflated.
315
316 // boxReg refers to the on-stack BasicLock in the current frame.
317 // We'd like to write:
318 // set box->_displaced_header = markWord::unused_mark(). Any non-0 value suffices.
319 // This is convenient but results a ST-before-CAS penalty. The following CAS suffers
320 // additional latency as we have another ST in the store buffer that must drain.
321
322 // avoid ST-before-CAS
323 // register juggle because we need tmpReg for cmpxchgptr below
324 movptr(scrReg, boxReg);
325 movptr(boxReg, tmpReg); // consider: LEA box, [tmp-2]
326
327 // Optimistic form: consider XORL tmpReg,tmpReg
328 movptr(tmpReg, NULL_WORD);
329
330 // Appears unlocked - try to swing _owner from null to non-null.
331 // Ideally, I'd manifest "Self" with get_thread and then attempt
332 // to CAS the register containing thread id into m->Owner.
333 // But we don't have enough registers, so instead we can either try to CAS
334 // rsp or the address of the box (in scr) into &m->owner. If the CAS succeeds
335 // we later store thread id into m->Owner. Transiently storing a stack address
336 // (rsp or the address of the box) into m->owner is harmless.
337 // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand.
338 lock();
339 cmpxchgptr(scrReg, Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
340 movptr(Address(scrReg, 0), 3); // box->_displaced_header = 3
341 // If we weren't able to swing _owner from null to the BasicLock
342 // then take the slow path.
343 jccb (Assembler::notZero, NO_COUNT);
344 // update _owner from BasicLock to thread
345 get_thread (scrReg); // beware: clobbers ICCs
346 movptr(scrReg, Address(scrReg, JavaThread::lock_id_offset()));
347 movptr(Address(boxReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)), scrReg);
348 xorptr(boxReg, boxReg); // set icc.ZFlag = 1 to indicate success
349
350 // If the CAS fails we can either retry or pass control to the slow path.
351 // We use the latter tactic.
352 // Pass the CAS result in the icc.ZFlag into DONE_LABEL
353 // If the CAS was successful ...
354 // Self has acquired the lock
355 // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it.
356 // Intentional fall-through into DONE_LABEL ...
357 #else // _LP64
358 // Unconditionally set box->_displaced_header = markWord::unused_mark().
359 // Without cast to int32_t this style of movptr will destroy r10 which is typically obj.
360 movptr(Address(boxReg, 0), checked_cast<int32_t>(markWord::unused_mark().value()));
361
362 // It's inflated and we use scrReg for ObjectMonitor* in this section.
363 movq(scrReg, tmpReg);
364 xorq(tmpReg, tmpReg);
365 movptr(boxReg, Address(r15_thread, JavaThread::lock_id_offset()));
366 lock();
367 cmpxchgptr(boxReg, Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(owner)));
368
369 // Propagate ICC.ZF from CAS above into DONE_LABEL.
370 jccb(Assembler::equal, COUNT); // CAS above succeeded; propagate ZF = 1 (success)
371
372 cmpptr(boxReg, rax); // Check if we are already the owner (recursive lock)
373 jccb(Assembler::notEqual, NO_COUNT); // If not recursive, ZF = 0 at this point (fail)
374 incq(Address(scrReg, OM_OFFSET_NO_MONITOR_VALUE_TAG(recursions)));
375 xorq(rax, rax); // Set ZF = 1 (success) for recursive lock, denoting locking success
376 #endif // _LP64
377 bind(DONE_LABEL);
378
379 // ZFlag == 1 count in fast path
380 // ZFlag == 0 count in slow path
381 jccb(Assembler::notZero, NO_COUNT); // jump if ZFlag == 0
382
383 bind(COUNT);
384 if (LockingMode == LM_LEGACY) {
385 #ifdef _LP64
386 // Count monitors in fast path
387 increment(Address(thread, JavaThread::held_monitor_count_offset()));
388 #endif
389 }
390 xorl(tmpReg, tmpReg); // Set ZF == 1
391
392 bind(NO_COUNT);
393
394 // At NO_COUNT the icc ZFlag is set as follows ...
395 // fast_unlock uses the same protocol.
396 // ZFlag == 1 -> Success
397 // ZFlag == 0 -> Failure - force control through the slow path
398 }
399
400 // obj: object to unlock
401 // box: box address (displaced header location), killed. Must be EAX.
402 // tmp: killed, cannot be obj nor box.
403 //
404 // Some commentary on balanced locking:
405 //
406 // fast_lock and fast_unlock are emitted only for provably balanced lock sites.
407 // Methods that don't have provably balanced locking are forced to run in the
408 // interpreter - such methods won't be compiled to use fast_lock and fast_unlock.
409 // The interpreter provides two properties:
508
509 bind (LSuccess);
510 testl (boxReg, 0); // set ICC.ZF=1 to indicate success
511 jmpb (DONE_LABEL);
512
513 if (LockingMode == LM_LEGACY) {
514 bind (Stacked);
515 movptr(tmpReg, Address (boxReg, 0)); // re-fetch
516 lock();
517 cmpxchgptr(tmpReg, Address(objReg, oopDesc::mark_offset_in_bytes())); // Uses RAX which is box
518 // Intentional fall-thru into DONE_LABEL
519 }
520
521 bind(DONE_LABEL);
522
523 // ZFlag == 1 count in fast path
524 // ZFlag == 0 count in slow path
525 jccb(Assembler::notZero, NO_COUNT);
526
527 bind(COUNT);
528
529 if (LockingMode == LM_LEGACY) {
530 // Count monitors in fast path
531 #ifdef _LP64
532 decrementq(Address(r15_thread, JavaThread::held_monitor_count_offset()));
533 #endif
534 }
535
536 xorl(tmpReg, tmpReg); // Set ZF == 1
537
538 bind(NO_COUNT);
539 }
540
541 void C2_MacroAssembler::fast_lock_lightweight(Register obj, Register box, Register rax_reg,
542 Register t, Register thread) {
543 assert(LockingMode == LM_LIGHTWEIGHT, "must be");
544 assert(rax_reg == rax, "Used for CAS");
545 assert_different_registers(obj, box, rax_reg, t, thread);
546
547 // Handle inflated monitor.
548 Label inflated;
549 // Finish fast lock successfully. ZF value is irrelevant.
550 Label locked;
551 // Finish fast lock unsuccessfully. MUST jump with ZF == 0
552 Label slow_path;
553
554 if (UseObjectMonitorTable) {
637 cmpptr(obj, Address(t));
638 jccb(Assembler::equal, monitor_found);
639
640 // Search until null encountered, guaranteed _null_sentinel at end.
641 cmpptr(Address(t), 1);
642 jcc(Assembler::below, slow_path); // 0 check, but with ZF=0 when *t == 0
643 increment(t, in_bytes(OMCache::oop_to_oop_difference()));
644 jmpb(loop);
645
646 // Cache hit.
647 bind(monitor_found);
648 movptr(monitor, Address(t, OMCache::oop_to_monitor_difference()));
649 }
650 const ByteSize monitor_tag = in_ByteSize(UseObjectMonitorTable ? 0 : checked_cast<int>(markWord::monitor_value));
651 const Address recursions_address(monitor, ObjectMonitor::recursions_offset() - monitor_tag);
652 const Address owner_address(monitor, ObjectMonitor::owner_offset() - monitor_tag);
653
654 Label monitor_locked;
655 // Lock the monitor.
656
657 if (UseObjectMonitorTable) {
658 // Cache the monitor for unlock before trashing box. On failure to acquire
659 // the lock, the slow path will reset the entry accordingly (see CacheSetter).
660 movptr(Address(box, BasicLock::object_monitor_cache_offset_in_bytes()), monitor);
661 }
662
663 // CAS owner (null => current thread).
664 xorptr(rax_reg, rax_reg);
665 movptr(box, Address(thread, JavaThread::lock_id_offset()));
666 lock(); cmpxchgptr(box, owner_address);
667 jccb(Assembler::equal, monitor_locked);
668
669 // Check if recursive.
670 cmpptr(box, rax_reg);
671 jccb(Assembler::notEqual, slow_path);
672
673 // Recursive.
674 increment(recursions_address);
675
676 bind(monitor_locked);
677 }
678
679 bind(locked);
680 // Set ZF = 1
681 xorl(rax_reg, rax_reg);
682
683 #ifdef ASSERT
684 // Check that locked label is reached with ZF set.
685 Label zf_correct;
686 Label zf_bad_zero;
687 jcc(Assembler::zero, zf_correct);
688 jmp(zf_bad_zero);
689 #endif
690
691 bind(slow_path);
692 #ifdef ASSERT
693 // Check that slow_path label is reached with ZF not set.
694 jcc(Assembler::notZero, zf_correct);
695 stop("Fast Lock ZF != 0");
696 bind(zf_bad_zero);
697 stop("Fast Lock ZF != 1");
698 bind(zf_correct);
699 #endif
823 // Check if there is a successor.
824 cmpptr(succ_address, NULL_WORD);
825 jccb(Assembler::notZero, unlocked); // If so we are done.
826
827 // Save the monitor pointer in the current thread, so we can try to
828 // reacquire the lock in SharedRuntime::monitor_exit_helper().
829 if (!UseObjectMonitorTable) {
830 andptr(monitor, ~(int32_t)markWord::monitor_value);
831 }
832 movptr(Address(thread, JavaThread::unlocked_inflated_monitor_offset()), monitor);
833
834 orl(t, 1); // Fast Unlock ZF = 0
835 jmpb(slow_path);
836
837 // Recursive unlock.
838 bind(recursive);
839 decrement(recursions_address);
840 }
841
842 bind(unlocked);
843 xorl(t, t); // Fast Unlock ZF = 1
844
845 #ifdef ASSERT
846 // Check that unlocked label is reached with ZF set.
847 Label zf_correct;
848 jcc(Assembler::zero, zf_correct);
849 stop("Fast Unlock ZF != 1");
850 #endif
851
852 bind(slow_path);
853 if (stub != nullptr) {
854 bind(stub->slow_path_continuation());
855 }
856 #ifdef ASSERT
857 // Check that stub->continuation() label is reached with ZF not set.
858 jccb(Assembler::notZero, zf_correct);
859 stop("Fast Unlock ZF != 0");
860 bind(zf_correct);
861 #endif
862 // C2 uses the value of ZF to determine the continuation.
|