2604
2605 and_(R0/*==0?*/, current_header, temp);
2606 // If condition is true we are cont and hence we can store 0 as the
2607 // displaced header in the box, which indicates that it is a recursive lock.
2608 std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), box);
2609
2610 if (flag != CCR0) {
2611 mcrf(flag, CCR0);
2612 }
2613 beq(CCR0, success);
2614 b(failure);
2615 }
2616
2617 // Handle existing monitor.
2618 bind(object_has_monitor);
2619 // The object's monitor m is unlocked iff m->owner is null,
2620 // otherwise m->owner may contain a thread or a stack address.
2621
2622 // Try to CAS m->owner from null to current thread.
2623 addi(temp, displaced_header, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value);
2624 cmpxchgd(/*flag=*/flag,
2625 /*current_value=*/current_header,
2626 /*compare_value=*/(intptr_t)0,
2627 /*exchange_value=*/R16_thread,
2628 /*where=*/temp,
2629 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2630 MacroAssembler::cmpxchgx_hint_acquire_lock());
2631
2632 // Store a non-null value into the box.
2633 std(box, BasicLock::displaced_header_offset_in_bytes(), box);
2634 beq(flag, success);
2635
2636 // Check for recursive locking.
2637 cmpd(flag, current_header, R16_thread);
2638 bne(flag, failure);
2639
2640 // Current thread already owns the lock. Just increment recursions.
2641 Register recursions = displaced_header;
2642 ld(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2643 addi(recursions, recursions, 1);
2644 std(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2645
2646 // flag == EQ indicates success, increment held monitor count
2647 // flag == NE indicates failure
2648 bind(success);
2649 inc_held_monitor_count(temp);
2650 bind(failure);
2651 }
2652
2653 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
2654 Register temp, Register displaced_header, Register current_header) {
2655 assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight");
2656 assert_different_registers(oop, box, temp, displaced_header, current_header);
2657 Label success, failure, object_has_monitor, notRecursive;
2684 /*current_value=*/current_header,
2685 /*compare_value=*/box,
2686 /*exchange_value=*/displaced_header,
2687 /*where=*/oop,
2688 MacroAssembler::MemBarRel,
2689 MacroAssembler::cmpxchgx_hint_release_lock(),
2690 noreg,
2691 &failure);
2692 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2693 b(success);
2694 }
2695
2696 // Handle existing monitor.
2697 bind(object_has_monitor);
2698 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
2699 addi(current_header, current_header, -(int)markWord::monitor_value); // monitor
2700 ld(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2701
2702 // In case of LM_LIGHTWEIGHT, we may reach here with (temp & ObjectMonitor::ANONYMOUS_OWNER) != 0.
2703 // This is handled like owner thread mismatches: We take the slow path.
2704 cmpd(flag, temp, R16_thread);
2705 bne(flag, failure);
2706
2707 ld(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2708
2709 addic_(displaced_header, displaced_header, -1);
2710 blt(CCR0, notRecursive); // Not recursive if negative after decrement.
2711 std(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2712 if (flag == CCR0) { // Otherwise, flag is already EQ, here.
2713 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Set CCR0 EQ
2714 }
2715 b(success);
2716
2717 bind(notRecursive);
2718
2719 // Set owner to null.
2720 // Release to satisfy the JMM
2721 release();
2722 li(temp, 0);
2723 std(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2724 // We need a full fence after clearing owner to avoid stranding.
2855
2856 // Check for match.
2857 ld(tmp3, 0, cache_addr);
2858 cmpd(CCR0, tmp3, obj);
2859 beq(CCR0, monitor_found);
2860
2861 // Search until null encountered, guaranteed _null_sentinel at end.
2862 addi(cache_addr, cache_addr, in_bytes(OMCache::oop_to_oop_difference()));
2863 cmpdi(CCR1, tmp3, 0);
2864 bne(CCR1, loop);
2865 // Cache Miss, CCR0.NE set from cmp above
2866 b(slow_path);
2867
2868 bind(monitor_found);
2869 ld(monitor, in_bytes(OMCache::oop_to_monitor_difference()), cache_addr);
2870
2871 // Compute owner address.
2872 addi(owner_addr, monitor, in_bytes(ObjectMonitor::owner_offset()));
2873 }
2874
2875 // CAS owner (null => current thread).
2876 cmpxchgd(/*flag=*/CCR0,
2877 /*current_value=*/t,
2878 /*compare_value=*/(intptr_t)0,
2879 /*exchange_value=*/R16_thread,
2880 /*where=*/owner_addr,
2881 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2882 MacroAssembler::cmpxchgx_hint_acquire_lock());
2883 beq(CCR0, monitor_locked);
2884
2885 // Check if recursive.
2886 cmpd(CCR0, t, R16_thread);
2887 bne(CCR0, slow_path);
2888
2889 // Recursive.
2890 if (!UseObjectMonitorTable) {
2891 assert_different_registers(tmp1, owner_addr);
2892 ld(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2893 addi(tmp1, tmp1, 1);
2894 std(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2895 } else {
2896 assert_different_registers(tmp2, monitor);
2897 ld(tmp2, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2898 addi(tmp2, tmp2, 1);
2899 std(tmp2, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2900 }
2901
2902 bind(monitor_locked);
2903 if (UseObjectMonitorTable) {
2904 std(monitor, BasicLock::object_monitor_cache_offset_in_bytes(), box);
2905 }
2906 }
|
2604
2605 and_(R0/*==0?*/, current_header, temp);
2606 // If condition is true we are cont and hence we can store 0 as the
2607 // displaced header in the box, which indicates that it is a recursive lock.
2608 std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), box);
2609
2610 if (flag != CCR0) {
2611 mcrf(flag, CCR0);
2612 }
2613 beq(CCR0, success);
2614 b(failure);
2615 }
2616
2617 // Handle existing monitor.
2618 bind(object_has_monitor);
2619 // The object's monitor m is unlocked iff m->owner is null,
2620 // otherwise m->owner may contain a thread or a stack address.
2621
2622 // Try to CAS m->owner from null to current thread.
2623 addi(temp, displaced_header, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value);
2624 Register thread_id = displaced_header;
2625 ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2626 cmpxchgd(/*flag=*/flag,
2627 /*current_value=*/current_header,
2628 /*compare_value=*/(intptr_t)0,
2629 /*exchange_value=*/thread_id,
2630 /*where=*/temp,
2631 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2632 MacroAssembler::cmpxchgx_hint_acquire_lock());
2633
2634 // Store a non-null value into the box.
2635 std(box, BasicLock::displaced_header_offset_in_bytes(), box);
2636 beq(flag, success);
2637
2638 // Check for recursive locking.
2639 cmpd(flag, current_header, thread_id);
2640 bne(flag, failure);
2641
2642 // Current thread already owns the lock. Just increment recursions.
2643 Register recursions = displaced_header;
2644 ld(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2645 addi(recursions, recursions, 1);
2646 std(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2647
2648 // flag == EQ indicates success, increment held monitor count
2649 // flag == NE indicates failure
2650 bind(success);
2651 inc_held_monitor_count(temp);
2652 bind(failure);
2653 }
2654
2655 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
2656 Register temp, Register displaced_header, Register current_header) {
2657 assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight");
2658 assert_different_registers(oop, box, temp, displaced_header, current_header);
2659 Label success, failure, object_has_monitor, notRecursive;
2686 /*current_value=*/current_header,
2687 /*compare_value=*/box,
2688 /*exchange_value=*/displaced_header,
2689 /*where=*/oop,
2690 MacroAssembler::MemBarRel,
2691 MacroAssembler::cmpxchgx_hint_release_lock(),
2692 noreg,
2693 &failure);
2694 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2695 b(success);
2696 }
2697
2698 // Handle existing monitor.
2699 bind(object_has_monitor);
2700 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
2701 addi(current_header, current_header, -(int)markWord::monitor_value); // monitor
2702 ld(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2703
2704 // In case of LM_LIGHTWEIGHT, we may reach here with (temp & ObjectMonitor::ANONYMOUS_OWNER) != 0.
2705 // This is handled like owner thread mismatches: We take the slow path.
2706 Register thread_id = displaced_header;
2707 ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2708 cmpd(flag, temp, thread_id);
2709 bne(flag, failure);
2710
2711 ld(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2712
2713 addic_(displaced_header, displaced_header, -1);
2714 blt(CCR0, notRecursive); // Not recursive if negative after decrement.
2715 std(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2716 if (flag == CCR0) { // Otherwise, flag is already EQ, here.
2717 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Set CCR0 EQ
2718 }
2719 b(success);
2720
2721 bind(notRecursive);
2722
2723 // Set owner to null.
2724 // Release to satisfy the JMM
2725 release();
2726 li(temp, 0);
2727 std(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2728 // We need a full fence after clearing owner to avoid stranding.
2859
2860 // Check for match.
2861 ld(tmp3, 0, cache_addr);
2862 cmpd(CCR0, tmp3, obj);
2863 beq(CCR0, monitor_found);
2864
2865 // Search until null encountered, guaranteed _null_sentinel at end.
2866 addi(cache_addr, cache_addr, in_bytes(OMCache::oop_to_oop_difference()));
2867 cmpdi(CCR1, tmp3, 0);
2868 bne(CCR1, loop);
2869 // Cache Miss, CCR0.NE set from cmp above
2870 b(slow_path);
2871
2872 bind(monitor_found);
2873 ld(monitor, in_bytes(OMCache::oop_to_monitor_difference()), cache_addr);
2874
2875 // Compute owner address.
2876 addi(owner_addr, monitor, in_bytes(ObjectMonitor::owner_offset()));
2877 }
2878
2879 // CAS owner (null => current thread id).
2880 Register thread_id = tmp1;
2881 ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2882 cmpxchgd(/*flag=*/CCR0,
2883 /*current_value=*/t,
2884 /*compare_value=*/(intptr_t)0,
2885 /*exchange_value=*/thread_id,
2886 /*where=*/owner_addr,
2887 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2888 MacroAssembler::cmpxchgx_hint_acquire_lock());
2889 beq(CCR0, monitor_locked);
2890
2891 // Check if recursive.
2892 cmpd(CCR0, t, thread_id);
2893 bne(CCR0, slow_path);
2894
2895 // Recursive.
2896 if (!UseObjectMonitorTable) {
2897 assert_different_registers(tmp1, owner_addr);
2898 ld(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2899 addi(tmp1, tmp1, 1);
2900 std(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2901 } else {
2902 assert_different_registers(tmp2, monitor);
2903 ld(tmp2, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2904 addi(tmp2, tmp2, 1);
2905 std(tmp2, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2906 }
2907
2908 bind(monitor_locked);
2909 if (UseObjectMonitorTable) {
2910 std(monitor, BasicLock::object_monitor_cache_offset_in_bytes(), box);
2911 }
2912 }
|