1 /*
2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
37 #include "memory/resourceArea.hpp"
38 #include "memory/universe.hpp"
39 #include "oops/accessDecorators.hpp"
40 #include "oops/compressedOops.inline.hpp"
41 #include "oops/klass.inline.hpp"
42 #include "prims/methodHandles.hpp"
43 #include "runtime/biasedLocking.hpp"
44 #include "runtime/flags/flagSetting.hpp"
45 #include "runtime/interfaceSupport.inline.hpp"
46 #include "runtime/jniHandles.hpp"
47 #include "runtime/objectMonitor.hpp"
48 #include "runtime/os.hpp"
49 #include "runtime/safepoint.hpp"
50 #include "runtime/safepointMechanism.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubRoutines.hpp"
53 #include "runtime/thread.hpp"
54 #include "utilities/macros.hpp"
55 #include "crc32c.h"
56
57 #ifdef PRODUCT
58 #define BLOCK_COMMENT(str) /* nothing */
59 #define STOP(error) stop(error)
60 #else
61 #define BLOCK_COMMENT(str) block_comment(str)
62 #define STOP(error) block_comment(error); stop(error)
63 #endif
64
65 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
66
67 #ifdef ASSERT
68 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
69 #endif
70
71 static Assembler::Condition reverse[] = {
72 Assembler::noOverflow /* overflow = 0x0 */ ,
73 Assembler::overflow /* noOverflow = 0x1 */ ,
74 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ ,
75 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ ,
76 Assembler::notZero /* zero = 0x4, equal = 0x4 */ ,
3754 Register t1,
3755 Register t2,
3756 Label& slow_case) {
3757 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3758 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3759 }
3760
3761 // Defines obj, preserves var_size_in_bytes
3762 void MacroAssembler::eden_allocate(Register thread, Register obj,
3763 Register var_size_in_bytes,
3764 int con_size_in_bytes,
3765 Register t1,
3766 Label& slow_case) {
3767 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3768 bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
3769 }
3770
3771 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
3772 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
3773 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
3774 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
3775 Label done;
3776
3777 testptr(length_in_bytes, length_in_bytes);
3778 jcc(Assembler::zero, done);
3779
3780 // initialize topmost word, divide index by 2, check if odd and test if zero
3781 // note: for the remaining code to work, index must be a multiple of BytesPerWord
3782 #ifdef ASSERT
3783 {
3784 Label L;
3785 testptr(length_in_bytes, BytesPerWord - 1);
3786 jcc(Assembler::zero, L);
3787 stop("length must be a multiple of BytesPerWord");
3788 bind(L);
3789 }
3790 #endif
3791 Register index = length_in_bytes;
3792 xorptr(temp, temp); // use _zero reg to clear memory (shorter code)
3793 if (UseIncDec) {
3794 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set
3795 } else {
3796 shrptr(index, 2); // use 2 instructions to avoid partial flag stall
3797 shrptr(index, 1);
3798 }
3799 #ifndef _LP64
3800 // index could have not been a multiple of 8 (i.e., bit 2 was set)
3801 {
3802 Label even;
3803 // note: if index was a multiple of 8, then it cannot
3804 // be 0 now otherwise it must have been 0 before
3805 // => if it is even, we don't need to check for 0 again
3806 jcc(Assembler::carryClear, even);
3807 // clear topmost word (no jump would be needed if conditional assignment worked here)
3808 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
3809 // index could be 0 now, must check again
3810 jcc(Assembler::zero, done);
3811 bind(even);
3812 }
4716
4717 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
4718 // get mirror
4719 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4720 load_method_holder(mirror, method);
4721 movptr(mirror, Address(mirror, mirror_offset));
4722 resolve_oop_handle(mirror, tmp);
4723 }
4724
4725 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4726 load_method_holder(rresult, rmethod);
4727 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4728 }
4729
4730 void MacroAssembler::load_method_holder(Register holder, Register method) {
4731 movptr(holder, Address(method, Method::const_offset())); // ConstMethod*
4732 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
4733 movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4734 }
4735
4736 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
4737 assert_different_registers(src, tmp);
4738 assert_different_registers(dst, tmp);
4739 #ifdef _LP64
4740 if (UseCompressedClassPointers) {
4741 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4742 decode_klass_not_null(dst, tmp);
4743 } else
4744 #endif
4745 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4746 }
4747
4748 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
4749 load_klass(dst, src, tmp);
4750 movptr(dst, Address(dst, Klass::prototype_header_offset()));
4751 }
4752
4753 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
4754 assert_different_registers(src, tmp);
4755 assert_different_registers(dst, tmp);
4756 #ifdef _LP64
4757 if (UseCompressedClassPointers) {
4758 encode_klass_not_null(src, tmp);
4759 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4760 } else
4761 #endif
4762 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4763 }
4764
4765 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
4766 Register tmp1, Register thread_tmp) {
4767 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4768 decorators = AccessInternal::decorator_fixup(decorators);
4769 bool as_raw = (decorators & AS_RAW) != 0;
4770 if (as_raw) {
4771 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4772 } else {
4773 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4774 }
4775 }
4776
4777 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
4778 Register tmp1, Register tmp2) {
4779 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4780 decorators = AccessInternal::decorator_fixup(decorators);
4781 bool as_raw = (decorators & AS_RAW) != 0;
4782 if (as_raw) {
8669
8670 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
8671
8672 #ifdef _LP64
8673 pop(r11);
8674 pop(r10);
8675 pop(r9);
8676 pop(r8);
8677 #endif
8678 pop(rcx);
8679 pop(rdx);
8680 LP64_ONLY(pop(rsi);)
8681 LP64_ONLY(pop(rdi);)
8682 if (thread != rax) {
8683 mov(thread, rax);
8684 pop(rax);
8685 }
8686 }
8687
8688 #endif // !WIN32 || _LP64
|
1 /*
2 * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation.
8 *
9 * This code is distributed in the hope that it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
37 #include "memory/resourceArea.hpp"
38 #include "memory/universe.hpp"
39 #include "oops/accessDecorators.hpp"
40 #include "oops/compressedOops.inline.hpp"
41 #include "oops/klass.inline.hpp"
42 #include "prims/methodHandles.hpp"
43 #include "runtime/biasedLocking.hpp"
44 #include "runtime/flags/flagSetting.hpp"
45 #include "runtime/interfaceSupport.inline.hpp"
46 #include "runtime/jniHandles.hpp"
47 #include "runtime/objectMonitor.hpp"
48 #include "runtime/os.hpp"
49 #include "runtime/safepoint.hpp"
50 #include "runtime/safepointMechanism.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubRoutines.hpp"
53 #include "runtime/thread.hpp"
54 #include "utilities/macros.hpp"
55 #include "crc32c.h"
56
57 #ifdef COMPILER2
58 #include "opto/c2_CodeStubs.hpp"
59 #include "opto/compile.hpp"
60 #include "opto/output.hpp"
61 #endif
62
63 #ifdef PRODUCT
64 #define BLOCK_COMMENT(str) /* nothing */
65 #define STOP(error) stop(error)
66 #else
67 #define BLOCK_COMMENT(str) block_comment(str)
68 #define STOP(error) block_comment(error); stop(error)
69 #endif
70
71 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
72
73 #ifdef ASSERT
74 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
75 #endif
76
77 static Assembler::Condition reverse[] = {
78 Assembler::noOverflow /* overflow = 0x0 */ ,
79 Assembler::overflow /* noOverflow = 0x1 */ ,
80 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ ,
81 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ ,
82 Assembler::notZero /* zero = 0x4, equal = 0x4 */ ,
3760 Register t1,
3761 Register t2,
3762 Label& slow_case) {
3763 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3764 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3765 }
3766
3767 // Defines obj, preserves var_size_in_bytes
3768 void MacroAssembler::eden_allocate(Register thread, Register obj,
3769 Register var_size_in_bytes,
3770 int con_size_in_bytes,
3771 Register t1,
3772 Label& slow_case) {
3773 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3774 bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
3775 }
3776
3777 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
3778 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
3779 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
3780 assert((offset_in_bytes & (BytesPerInt - 1)) == 0, "offset must be a multiple of BytesPerInt");
3781 Label done;
3782
3783 testptr(length_in_bytes, length_in_bytes);
3784 jcc(Assembler::zero, done);
3785
3786 // Emit single 32bit store to clear leading bytes, if necessary.
3787 xorptr(temp, temp); // use _zero reg to clear memory (shorter code)
3788 #ifdef _LP64
3789 if (!is_aligned(offset_in_bytes, BytesPerWord)) {
3790 movl(Address(address, offset_in_bytes), temp);
3791 offset_in_bytes += BytesPerInt;
3792 decrement(length_in_bytes, BytesPerInt);
3793 }
3794 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
3795 testptr(length_in_bytes, length_in_bytes);
3796 jcc(Assembler::zero, done);
3797 #endif
3798
3799 // initialize topmost word, divide index by 2, check if odd and test if zero
3800 // note: for the remaining code to work, index must be a multiple of BytesPerWord
3801 #ifdef ASSERT
3802 {
3803 Label L;
3804 testptr(length_in_bytes, BytesPerWord - 1);
3805 jcc(Assembler::zero, L);
3806 stop("length must be a multiple of BytesPerWord");
3807 bind(L);
3808 }
3809 #endif
3810 Register index = length_in_bytes;
3811 if (UseIncDec) {
3812 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set
3813 } else {
3814 shrptr(index, 2); // use 2 instructions to avoid partial flag stall
3815 shrptr(index, 1);
3816 }
3817 #ifndef _LP64
3818 // index could have not been a multiple of 8 (i.e., bit 2 was set)
3819 {
3820 Label even;
3821 // note: if index was a multiple of 8, then it cannot
3822 // be 0 now otherwise it must have been 0 before
3823 // => if it is even, we don't need to check for 0 again
3824 jcc(Assembler::carryClear, even);
3825 // clear topmost word (no jump would be needed if conditional assignment worked here)
3826 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
3827 // index could be 0 now, must check again
3828 jcc(Assembler::zero, done);
3829 bind(even);
3830 }
4734
4735 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
4736 // get mirror
4737 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4738 load_method_holder(mirror, method);
4739 movptr(mirror, Address(mirror, mirror_offset));
4740 resolve_oop_handle(mirror, tmp);
4741 }
4742
4743 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4744 load_method_holder(rresult, rmethod);
4745 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4746 }
4747
4748 void MacroAssembler::load_method_holder(Register holder, Register method) {
4749 movptr(holder, Address(method, Method::const_offset())); // ConstMethod*
4750 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
4751 movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4752 }
4753
4754 #ifdef _LP64
4755 void MacroAssembler::load_nklass(Register dst, Register src) {
4756 assert(UseCompressedClassPointers, "expect compressed class pointers");
4757
4758 if (!UseCompactObjectHeaders) {
4759 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4760 return;
4761 }
4762
4763 Label fast;
4764 movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
4765 testb(dst, markWord::monitor_value);
4766 jccb(Assembler::zero, fast);
4767
4768 // Fetch displaced header
4769 movq(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
4770
4771 bind(fast);
4772 shrq(dst, markWord::klass_shift);
4773 }
4774 #endif
4775
4776 void MacroAssembler::load_klass(Register dst, Register src, Register tmp, bool null_check_src) {
4777 assert_different_registers(src, tmp);
4778 assert_different_registers(dst, tmp);
4779 if (null_check_src) {
4780 if (UseCompactObjectHeaders) {
4781 null_check(src, oopDesc::mark_offset_in_bytes());
4782 } else {
4783 null_check(src, oopDesc::klass_offset_in_bytes());
4784 }
4785 }
4786 #ifdef _LP64
4787 if (UseCompressedClassPointers) {
4788 load_nklass(dst, src);
4789 decode_klass_not_null(dst, tmp);
4790 } else
4791 #endif
4792 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4793 }
4794
4795 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
4796 load_klass(dst, src, tmp);
4797 movptr(dst, Address(dst, Klass::prototype_header_offset()));
4798 }
4799
4800 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
4801 assert(!UseCompactObjectHeaders, "not with compact headers");
4802 assert_different_registers(src, tmp);
4803 assert_different_registers(dst, tmp);
4804 #ifdef _LP64
4805 if (UseCompressedClassPointers) {
4806 encode_klass_not_null(src, tmp);
4807 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4808 } else
4809 #endif
4810 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4811 }
4812
4813 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
4814 #ifdef _LP64
4815 if (UseCompactObjectHeaders) {
4816 // NOTE: We need to deal with possible ObjectMonitor in object header.
4817 // Eventually we might be able to do simple movl & cmpl like in
4818 // the CCP path below.
4819 load_nklass(tmp, obj);
4820 cmpl(klass, tmp);
4821 } else if (UseCompressedClassPointers) {
4822 cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
4823 } else
4824 #endif
4825 {
4826 cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
4827 }
4828 }
4829
4830 void MacroAssembler::cmp_klass(Register src, Register dst, Register tmp1, Register tmp2) {
4831 #ifdef _LP64
4832 if (UseCompactObjectHeaders) {
4833 // NOTE: We need to deal with possible ObjectMonitor in object header.
4834 // Eventually we might be able to do simple movl & cmpl like in
4835 // the CCP path below.
4836 assert(tmp2 != noreg, "need tmp2");
4837 assert_different_registers(src, dst, tmp1, tmp2);
4838 load_nklass(tmp1, src);
4839 load_nklass(tmp2, dst);
4840 cmpl(tmp1, tmp2);
4841 } else if (UseCompressedClassPointers) {
4842 movl(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
4843 cmpl(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
4844 } else
4845 #endif
4846 {
4847 movptr(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
4848 cmpptr(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
4849 }
4850 }
4851
4852 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
4853 Register tmp1, Register thread_tmp) {
4854 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4855 decorators = AccessInternal::decorator_fixup(decorators);
4856 bool as_raw = (decorators & AS_RAW) != 0;
4857 if (as_raw) {
4858 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4859 } else {
4860 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4861 }
4862 }
4863
4864 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
4865 Register tmp1, Register tmp2) {
4866 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4867 decorators = AccessInternal::decorator_fixup(decorators);
4868 bool as_raw = (decorators & AS_RAW) != 0;
4869 if (as_raw) {
8756
8757 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
8758
8759 #ifdef _LP64
8760 pop(r11);
8761 pop(r10);
8762 pop(r9);
8763 pop(r8);
8764 #endif
8765 pop(rcx);
8766 pop(rdx);
8767 LP64_ONLY(pop(rsi);)
8768 LP64_ONLY(pop(rdi);)
8769 if (thread != rax) {
8770 mov(thread, rax);
8771 pop(rax);
8772 }
8773 }
8774
8775 #endif // !WIN32 || _LP64
8776
8777 // Implements lightweight-locking.
8778 //
8779 // obj: the object to be locked
8780 // reg_rax: rax
8781 // thread: the thread which attempts to lock obj
8782 // tmp: a temporary register
8783 void MacroAssembler::lightweight_lock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
8784 assert(reg_rax == rax, "");
8785 assert_different_registers(obj, reg_rax, thread, tmp);
8786
8787 Label push;
8788 const Register top = tmp;
8789
8790 // Preload the markWord. It is important that this is the first
8791 // instruction emitted as it is part of C1's null check semantics.
8792 movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
8793
8794 // Load top.
8795 movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
8796
8797 // Check if the lock-stack is full.
8798 cmpl(top, LockStack::end_offset());
8799 jcc(Assembler::greaterEqual, slow);
8800
8801 // Check for recursion.
8802 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
8803 jcc(Assembler::equal, push);
8804
8805 // Check header for monitor (0b10).
8806 testptr(reg_rax, markWord::monitor_value);
8807 jcc(Assembler::notZero, slow);
8808
8809 // Try to lock. Transition lock bits 0b01 => 0b00
8810 movptr(tmp, reg_rax);
8811 andptr(tmp, ~(int32_t)markWord::unlocked_value);
8812 orptr(reg_rax, markWord::unlocked_value);
8813 lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
8814 jcc(Assembler::notEqual, slow);
8815
8816 // Restore top, CAS clobbers register.
8817 movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
8818
8819 bind(push);
8820 // After successful lock, push object on lock-stack.
8821 movptr(Address(thread, top), obj);
8822 incrementl(top, oopSize);
8823 movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
8824 }
8825
8826 // Implements lightweight-unlocking.
8827 //
8828 // obj: the object to be unlocked
8829 // reg_rax: rax
8830 // thread: the thread
8831 // tmp: a temporary register
8832 //
8833 // x86_32 Note: reg_rax and thread may alias each other due to limited register
8834 // availiability.
8835 void MacroAssembler::lightweight_unlock(Register obj, Register reg_rax, Register thread, Register tmp, Label& slow) {
8836 assert(reg_rax == rax, "");
8837 assert_different_registers(obj, reg_rax, tmp);
8838 LP64_ONLY(assert_different_registers(obj, reg_rax, thread, tmp);)
8839
8840 Label unlocked, push_and_slow;
8841 const Register top = tmp;
8842
8843 // Check if obj is top of lock-stack.
8844 movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
8845 cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
8846 jcc(Assembler::notEqual, slow);
8847
8848 // Pop lock-stack.
8849 DEBUG_ONLY(movptr(Address(thread, top, Address::times_1, -oopSize), 0);)
8850 subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
8851
8852 // Check if recursive.
8853 cmpptr(obj, Address(thread, top, Address::times_1, -2 * oopSize));
8854 jcc(Assembler::equal, unlocked);
8855
8856 // Not recursive. Check header for monitor (0b10).
8857 movptr(reg_rax, Address(obj, oopDesc::mark_offset_in_bytes()));
8858 testptr(reg_rax, markWord::monitor_value);
8859 jcc(Assembler::notZero, push_and_slow);
8860
8861 #ifdef ASSERT
8862 // Check header not unlocked (0b01).
8863 Label not_unlocked;
8864 testptr(reg_rax, markWord::unlocked_value);
8865 jcc(Assembler::zero, not_unlocked);
8866 stop("lightweight_unlock already unlocked");
8867 bind(not_unlocked);
8868 #endif
8869
8870 // Try to unlock. Transition lock bits 0b00 => 0b01
8871 movptr(tmp, reg_rax);
8872 orptr(tmp, markWord::unlocked_value);
8873 lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
8874 jcc(Assembler::equal, unlocked);
8875
8876 bind(push_and_slow);
8877 // Restore lock-stack and handle the unlock in runtime.
8878 if (thread == reg_rax) {
8879 // On x86_32 we may lose the thread.
8880 get_thread(thread);
8881 }
8882 #ifdef ASSERT
8883 movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
8884 movptr(Address(thread, top), obj);
8885 #endif
8886 addl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
8887 jmp(slow);
8888
8889 bind(unlocked);
8890 }
|