2611
2612 and_(R0/*==0?*/, current_header, temp);
2613 // If condition is true we are cont and hence we can store 0 as the
2614 // displaced header in the box, which indicates that it is a recursive lock.
2615 std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), box);
2616
2617 if (flag != CCR0) {
2618 mcrf(flag, CCR0);
2619 }
2620 beq(CCR0, success);
2621 b(failure);
2622 }
2623
2624 // Handle existing monitor.
2625 bind(object_has_monitor);
2626 // The object's monitor m is unlocked iff m->owner is null,
2627 // otherwise m->owner may contain a thread or a stack address.
2628
2629 // Try to CAS m->owner from null to current thread.
2630 addi(temp, displaced_header, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value);
2631 cmpxchgd(/*flag=*/flag,
2632 /*current_value=*/current_header,
2633 /*compare_value=*/(intptr_t)0,
2634 /*exchange_value=*/R16_thread,
2635 /*where=*/temp,
2636 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2637 MacroAssembler::cmpxchgx_hint_acquire_lock());
2638
2639 // Store a non-null value into the box.
2640 std(box, BasicLock::displaced_header_offset_in_bytes(), box);
2641 beq(flag, success);
2642
2643 // Check for recursive locking.
2644 cmpd(flag, current_header, R16_thread);
2645 bne(flag, failure);
2646
2647 // Current thread already owns the lock. Just increment recursions.
2648 Register recursions = displaced_header;
2649 ld(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2650 addi(recursions, recursions, 1);
2651 std(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2652
2653 // flag == EQ indicates success, increment held monitor count
2654 // flag == NE indicates failure
2655 bind(success);
2656 inc_held_monitor_count(temp);
2657 bind(failure);
2658 }
2659
2660 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
2661 Register temp, Register displaced_header, Register current_header) {
2662 assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight");
2663 assert_different_registers(oop, box, temp, displaced_header, current_header);
2664 Label success, failure, object_has_monitor, notRecursive;
2691 /*current_value=*/current_header,
2692 /*compare_value=*/box,
2693 /*exchange_value=*/displaced_header,
2694 /*where=*/oop,
2695 MacroAssembler::MemBarRel,
2696 MacroAssembler::cmpxchgx_hint_release_lock(),
2697 noreg,
2698 &failure);
2699 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2700 b(success);
2701 }
2702
2703 // Handle existing monitor.
2704 bind(object_has_monitor);
2705 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
2706 addi(current_header, current_header, -(int)markWord::monitor_value); // monitor
2707 ld(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2708
2709 // In case of LM_LIGHTWEIGHT, we may reach here with (temp & ObjectMonitor::ANONYMOUS_OWNER) != 0.
2710 // This is handled like owner thread mismatches: We take the slow path.
2711 cmpd(flag, temp, R16_thread);
2712 bne(flag, failure);
2713
2714 ld(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2715
2716 addic_(displaced_header, displaced_header, -1);
2717 blt(CCR0, notRecursive); // Not recursive if negative after decrement.
2718 std(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2719 if (flag == CCR0) { // Otherwise, flag is already EQ, here.
2720 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Set CCR0 EQ
2721 }
2722 b(success);
2723
2724 bind(notRecursive);
2725 ld(temp, in_bytes(ObjectMonitor::EntryList_offset()), current_header);
2726 ld(displaced_header, in_bytes(ObjectMonitor::cxq_offset()), current_header);
2727 orr(temp, temp, displaced_header); // Will be 0 if both are 0.
2728 cmpdi(flag, temp, 0);
2729 bne(flag, failure);
2730 release();
2731 std(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2795
2796 bind(push);
2797 // After successful lock, push object on lock-stack.
2798 stdx(obj, R16_thread, top);
2799 addi(top, top, oopSize);
2800 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2801 b(locked);
2802 }
2803
2804 { // Handle inflated monitor.
2805 bind(inflated);
2806
2807 // mark contains the tagged ObjectMonitor*.
2808 const Register tagged_monitor = mark;
2809 const uintptr_t monitor_tag = markWord::monitor_value;
2810 const Register owner_addr = tmp2;
2811
2812 // Compute owner address.
2813 addi(owner_addr, tagged_monitor, in_bytes(ObjectMonitor::owner_offset()) - monitor_tag);
2814
2815 // CAS owner (null => current thread).
2816 cmpxchgd(/*flag=*/flag,
2817 /*current_value=*/t,
2818 /*compare_value=*/(intptr_t)0,
2819 /*exchange_value=*/R16_thread,
2820 /*where=*/owner_addr,
2821 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2822 MacroAssembler::cmpxchgx_hint_acquire_lock());
2823 beq(flag, locked);
2824
2825 // Check if recursive.
2826 cmpd(flag, t, R16_thread);
2827 bne(flag, slow_path);
2828
2829 // Recursive.
2830 ld(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2831 addi(tmp1, tmp1, 1);
2832 std(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2833 }
2834
2835 bind(locked);
2836 inc_held_monitor_count(tmp1);
2837
2838 #ifdef ASSERT
2839 // Check that locked label is reached with flag == EQ.
2840 Label flag_correct;
2841 beq(flag, flag_correct);
2842 stop("Fast Lock Flag != EQ");
2843 #endif
2844 bind(slow_path);
2845 #ifdef ASSERT
2846 // Check that slow_path label is reached with flag == NE.
2961 // Recursive unlock.
2962 std(recursions, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2963 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal);
2964 b(unlocked);
2965
2966 bind(not_recursive);
2967
2968 Label release_;
2969 const Register t2 = tmp2;
2970
2971 // Check if the entry lists are empty.
2972 ld(t, in_bytes(ObjectMonitor::EntryList_offset()), monitor);
2973 ld(t2, in_bytes(ObjectMonitor::cxq_offset()), monitor);
2974 orr(t, t, t2);
2975 cmpdi(flag, t, 0);
2976 beq(flag, release_);
2977
2978 // The owner may be anonymous and we removed the last obj entry in
2979 // the lock-stack. This loses the information about the owner.
2980 // Write the thread to the owner field so the runtime knows the owner.
2981 std(R16_thread, in_bytes(ObjectMonitor::owner_offset()), monitor);
2982 b(slow_path);
2983
2984 bind(release_);
2985 // Set owner to null.
2986 release();
2987 // t contains 0
2988 std(t, in_bytes(ObjectMonitor::owner_offset()), monitor);
2989 }
2990
2991 bind(unlocked);
2992 dec_held_monitor_count(t);
2993
2994 #ifdef ASSERT
2995 // Check that unlocked label is reached with flag == EQ.
2996 Label flag_correct;
2997 beq(flag, flag_correct);
2998 stop("Fast Lock Flag != EQ");
2999 #endif
3000 bind(slow_path);
3001 #ifdef ASSERT
|
2611
2612 and_(R0/*==0?*/, current_header, temp);
2613 // If condition is true we are cont and hence we can store 0 as the
2614 // displaced header in the box, which indicates that it is a recursive lock.
2615 std(R0/*==0, perhaps*/, BasicLock::displaced_header_offset_in_bytes(), box);
2616
2617 if (flag != CCR0) {
2618 mcrf(flag, CCR0);
2619 }
2620 beq(CCR0, success);
2621 b(failure);
2622 }
2623
2624 // Handle existing monitor.
2625 bind(object_has_monitor);
2626 // The object's monitor m is unlocked iff m->owner is null,
2627 // otherwise m->owner may contain a thread or a stack address.
2628
2629 // Try to CAS m->owner from null to current thread.
2630 addi(temp, displaced_header, in_bytes(ObjectMonitor::owner_offset()) - markWord::monitor_value);
2631 Register thread_id = displaced_header;
2632 ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2633 cmpxchgd(/*flag=*/flag,
2634 /*current_value=*/current_header,
2635 /*compare_value=*/(intptr_t)0,
2636 /*exchange_value=*/thread_id,
2637 /*where=*/temp,
2638 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2639 MacroAssembler::cmpxchgx_hint_acquire_lock());
2640
2641 // Store a non-null value into the box.
2642 std(box, BasicLock::displaced_header_offset_in_bytes(), box);
2643 beq(flag, success);
2644
2645 // Check for recursive locking.
2646 cmpd(flag, current_header, thread_id);
2647 bne(flag, failure);
2648
2649 // Current thread already owns the lock. Just increment recursions.
2650 Register recursions = displaced_header;
2651 ld(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2652 addi(recursions, recursions, 1);
2653 std(recursions, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), temp);
2654
2655 // flag == EQ indicates success, increment held monitor count
2656 // flag == NE indicates failure
2657 bind(success);
2658 inc_held_monitor_count(temp);
2659 bind(failure);
2660 }
2661
2662 void MacroAssembler::compiler_fast_unlock_object(ConditionRegister flag, Register oop, Register box,
2663 Register temp, Register displaced_header, Register current_header) {
2664 assert(LockingMode != LM_LIGHTWEIGHT, "uses fast_unlock_lightweight");
2665 assert_different_registers(oop, box, temp, displaced_header, current_header);
2666 Label success, failure, object_has_monitor, notRecursive;
2693 /*current_value=*/current_header,
2694 /*compare_value=*/box,
2695 /*exchange_value=*/displaced_header,
2696 /*where=*/oop,
2697 MacroAssembler::MemBarRel,
2698 MacroAssembler::cmpxchgx_hint_release_lock(),
2699 noreg,
2700 &failure);
2701 assert(oopDesc::mark_offset_in_bytes() == 0, "offset of _mark is not 0");
2702 b(success);
2703 }
2704
2705 // Handle existing monitor.
2706 bind(object_has_monitor);
2707 STATIC_ASSERT(markWord::monitor_value <= INT_MAX);
2708 addi(current_header, current_header, -(int)markWord::monitor_value); // monitor
2709 ld(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2710
2711 // In case of LM_LIGHTWEIGHT, we may reach here with (temp & ObjectMonitor::ANONYMOUS_OWNER) != 0.
2712 // This is handled like owner thread mismatches: We take the slow path.
2713 Register thread_id = displaced_header;
2714 ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2715 cmpd(flag, temp, thread_id);
2716 bne(flag, failure);
2717
2718 ld(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2719
2720 addic_(displaced_header, displaced_header, -1);
2721 blt(CCR0, notRecursive); // Not recursive if negative after decrement.
2722 std(displaced_header, in_bytes(ObjectMonitor::recursions_offset()), current_header);
2723 if (flag == CCR0) { // Otherwise, flag is already EQ, here.
2724 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal); // Set CCR0 EQ
2725 }
2726 b(success);
2727
2728 bind(notRecursive);
2729 ld(temp, in_bytes(ObjectMonitor::EntryList_offset()), current_header);
2730 ld(displaced_header, in_bytes(ObjectMonitor::cxq_offset()), current_header);
2731 orr(temp, temp, displaced_header); // Will be 0 if both are 0.
2732 cmpdi(flag, temp, 0);
2733 bne(flag, failure);
2734 release();
2735 std(temp, in_bytes(ObjectMonitor::owner_offset()), current_header);
2799
2800 bind(push);
2801 // After successful lock, push object on lock-stack.
2802 stdx(obj, R16_thread, top);
2803 addi(top, top, oopSize);
2804 stw(top, in_bytes(JavaThread::lock_stack_top_offset()), R16_thread);
2805 b(locked);
2806 }
2807
2808 { // Handle inflated monitor.
2809 bind(inflated);
2810
2811 // mark contains the tagged ObjectMonitor*.
2812 const Register tagged_monitor = mark;
2813 const uintptr_t monitor_tag = markWord::monitor_value;
2814 const Register owner_addr = tmp2;
2815
2816 // Compute owner address.
2817 addi(owner_addr, tagged_monitor, in_bytes(ObjectMonitor::owner_offset()) - monitor_tag);
2818
2819 // CAS owner (null => current thread id).
2820 Register thread_id = tmp1;
2821 ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2822 cmpxchgd(/*flag=*/flag,
2823 /*current_value=*/t,
2824 /*compare_value=*/(intptr_t)0,
2825 /*exchange_value=*/thread_id,
2826 /*where=*/owner_addr,
2827 MacroAssembler::MemBarRel | MacroAssembler::MemBarAcq,
2828 MacroAssembler::cmpxchgx_hint_acquire_lock());
2829 beq(flag, locked);
2830
2831 // Check if recursive.
2832 cmpd(flag, t, thread_id);
2833 bne(flag, slow_path);
2834
2835 // Recursive.
2836 ld(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2837 addi(tmp1, tmp1, 1);
2838 std(tmp1, in_bytes(ObjectMonitor::recursions_offset() - ObjectMonitor::owner_offset()), owner_addr);
2839 }
2840
2841 bind(locked);
2842 inc_held_monitor_count(tmp1);
2843
2844 #ifdef ASSERT
2845 // Check that locked label is reached with flag == EQ.
2846 Label flag_correct;
2847 beq(flag, flag_correct);
2848 stop("Fast Lock Flag != EQ");
2849 #endif
2850 bind(slow_path);
2851 #ifdef ASSERT
2852 // Check that slow_path label is reached with flag == NE.
2967 // Recursive unlock.
2968 std(recursions, in_bytes(ObjectMonitor::recursions_offset()), monitor);
2969 crorc(CCR0, Assembler::equal, CCR0, Assembler::equal);
2970 b(unlocked);
2971
2972 bind(not_recursive);
2973
2974 Label release_;
2975 const Register t2 = tmp2;
2976
2977 // Check if the entry lists are empty.
2978 ld(t, in_bytes(ObjectMonitor::EntryList_offset()), monitor);
2979 ld(t2, in_bytes(ObjectMonitor::cxq_offset()), monitor);
2980 orr(t, t, t2);
2981 cmpdi(flag, t, 0);
2982 beq(flag, release_);
2983
2984 // The owner may be anonymous and we removed the last obj entry in
2985 // the lock-stack. This loses the information about the owner.
2986 // Write the thread to the owner field so the runtime knows the owner.
2987 Register thread_id = tmp2;
2988 ld(thread_id, in_bytes(JavaThread::lock_id_offset()), R16_thread);
2989 std(thread_id, in_bytes(ObjectMonitor::owner_offset()), monitor);
2990 b(slow_path);
2991
2992 bind(release_);
2993 // Set owner to null.
2994 release();
2995 // t contains 0
2996 std(t, in_bytes(ObjectMonitor::owner_offset()), monitor);
2997 }
2998
2999 bind(unlocked);
3000 dec_held_monitor_count(t);
3001
3002 #ifdef ASSERT
3003 // Check that unlocked label is reached with flag == EQ.
3004 Label flag_correct;
3005 beq(flag, flag_correct);
3006 stop("Fast Lock Flag != EQ");
3007 #endif
3008 bind(slow_path);
3009 #ifdef ASSERT
|