37 #include "memory/resourceArea.hpp"
38 #include "memory/universe.hpp"
39 #include "oops/accessDecorators.hpp"
40 #include "oops/compressedOops.inline.hpp"
41 #include "oops/klass.inline.hpp"
42 #include "prims/methodHandles.hpp"
43 #include "runtime/biasedLocking.hpp"
44 #include "runtime/flags/flagSetting.hpp"
45 #include "runtime/interfaceSupport.inline.hpp"
46 #include "runtime/jniHandles.hpp"
47 #include "runtime/objectMonitor.hpp"
48 #include "runtime/os.hpp"
49 #include "runtime/safepoint.hpp"
50 #include "runtime/safepointMechanism.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubRoutines.hpp"
53 #include "runtime/thread.hpp"
54 #include "utilities/macros.hpp"
55 #include "crc32c.h"
56
57 #ifdef PRODUCT
58 #define BLOCK_COMMENT(str) /* nothing */
59 #define STOP(error) stop(error)
60 #else
61 #define BLOCK_COMMENT(str) block_comment(str)
62 #define STOP(error) block_comment(error); stop(error)
63 #endif
64
65 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
66
67 #ifdef ASSERT
68 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
69 #endif
70
71 static Assembler::Condition reverse[] = {
72 Assembler::noOverflow /* overflow = 0x0 */ ,
73 Assembler::overflow /* noOverflow = 0x1 */ ,
74 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ ,
75 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ ,
76 Assembler::notZero /* zero = 0x4, equal = 0x4 */ ,
3754 Register t1,
3755 Register t2,
3756 Label& slow_case) {
3757 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3758 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3759 }
3760
3761 // Defines obj, preserves var_size_in_bytes
3762 void MacroAssembler::eden_allocate(Register thread, Register obj,
3763 Register var_size_in_bytes,
3764 int con_size_in_bytes,
3765 Register t1,
3766 Label& slow_case) {
3767 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3768 bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
3769 }
3770
3771 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
3772 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
3773 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
3774 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
3775 Label done;
3776
3777 testptr(length_in_bytes, length_in_bytes);
3778 jcc(Assembler::zero, done);
3779
3780 // initialize topmost word, divide index by 2, check if odd and test if zero
3781 // note: for the remaining code to work, index must be a multiple of BytesPerWord
3782 #ifdef ASSERT
3783 {
3784 Label L;
3785 testptr(length_in_bytes, BytesPerWord - 1);
3786 jcc(Assembler::zero, L);
3787 stop("length must be a multiple of BytesPerWord");
3788 bind(L);
3789 }
3790 #endif
3791 Register index = length_in_bytes;
3792 xorptr(temp, temp); // use _zero reg to clear memory (shorter code)
3793 if (UseIncDec) {
3794 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set
3795 } else {
3796 shrptr(index, 2); // use 2 instructions to avoid partial flag stall
3797 shrptr(index, 1);
3798 }
3799 #ifndef _LP64
3800 // index could have not been a multiple of 8 (i.e., bit 2 was set)
3801 {
3802 Label even;
3803 // note: if index was a multiple of 8, then it cannot
3804 // be 0 now otherwise it must have been 0 before
3805 // => if it is even, we don't need to check for 0 again
3806 jcc(Assembler::carryClear, even);
3807 // clear topmost word (no jump would be needed if conditional assignment worked here)
3808 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
3809 // index could be 0 now, must check again
3810 jcc(Assembler::zero, done);
3811 bind(even);
3812 }
4716
4717 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
4718 // get mirror
4719 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4720 load_method_holder(mirror, method);
4721 movptr(mirror, Address(mirror, mirror_offset));
4722 resolve_oop_handle(mirror, tmp);
4723 }
4724
4725 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4726 load_method_holder(rresult, rmethod);
4727 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4728 }
4729
4730 void MacroAssembler::load_method_holder(Register holder, Register method) {
4731 movptr(holder, Address(method, Method::const_offset())); // ConstMethod*
4732 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
4733 movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4734 }
4735
4736 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
4737 assert_different_registers(src, tmp);
4738 assert_different_registers(dst, tmp);
4739 #ifdef _LP64
4740 if (UseCompressedClassPointers) {
4741 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4742 decode_klass_not_null(dst, tmp);
4743 } else
4744 #endif
4745 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4746 }
4747
4748 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
4749 load_klass(dst, src, tmp);
4750 movptr(dst, Address(dst, Klass::prototype_header_offset()));
4751 }
4752
4753 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
4754 assert_different_registers(src, tmp);
4755 assert_different_registers(dst, tmp);
4756 #ifdef _LP64
4757 if (UseCompressedClassPointers) {
4758 encode_klass_not_null(src, tmp);
4759 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4760 } else
4761 #endif
4762 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4763 }
4764
4765 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
4766 Register tmp1, Register thread_tmp) {
4767 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4768 decorators = AccessInternal::decorator_fixup(decorators);
4769 bool as_raw = (decorators & AS_RAW) != 0;
4770 if (as_raw) {
4771 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4772 } else {
4773 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4774 }
4775 }
4776
4777 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
4778 Register tmp1, Register tmp2) {
4779 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4780 decorators = AccessInternal::decorator_fixup(decorators);
4781 bool as_raw = (decorators & AS_RAW) != 0;
4782 if (as_raw) {
8666
8667 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
8668
8669 #ifdef _LP64
8670 pop(r11);
8671 pop(r10);
8672 pop(r9);
8673 pop(r8);
8674 #endif
8675 pop(rcx);
8676 pop(rdx);
8677 LP64_ONLY(pop(rsi);)
8678 LP64_ONLY(pop(rdi);)
8679 if (thread != rax) {
8680 mov(thread, rax);
8681 pop(rax);
8682 }
8683 }
8684
8685 #endif // !WIN32 || _LP64
|
37 #include "memory/resourceArea.hpp"
38 #include "memory/universe.hpp"
39 #include "oops/accessDecorators.hpp"
40 #include "oops/compressedOops.inline.hpp"
41 #include "oops/klass.inline.hpp"
42 #include "prims/methodHandles.hpp"
43 #include "runtime/biasedLocking.hpp"
44 #include "runtime/flags/flagSetting.hpp"
45 #include "runtime/interfaceSupport.inline.hpp"
46 #include "runtime/jniHandles.hpp"
47 #include "runtime/objectMonitor.hpp"
48 #include "runtime/os.hpp"
49 #include "runtime/safepoint.hpp"
50 #include "runtime/safepointMechanism.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubRoutines.hpp"
53 #include "runtime/thread.hpp"
54 #include "utilities/macros.hpp"
55 #include "crc32c.h"
56
57 #ifdef COMPILER2
58 #include "opto/c2_CodeStubs.hpp"
59 #include "opto/compile.hpp"
60 #include "opto/output.hpp"
61 #endif
62
63 #ifdef PRODUCT
64 #define BLOCK_COMMENT(str) /* nothing */
65 #define STOP(error) stop(error)
66 #else
67 #define BLOCK_COMMENT(str) block_comment(str)
68 #define STOP(error) block_comment(error); stop(error)
69 #endif
70
71 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
72
73 #ifdef ASSERT
74 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
75 #endif
76
77 static Assembler::Condition reverse[] = {
78 Assembler::noOverflow /* overflow = 0x0 */ ,
79 Assembler::overflow /* noOverflow = 0x1 */ ,
80 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ ,
81 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ ,
82 Assembler::notZero /* zero = 0x4, equal = 0x4 */ ,
3760 Register t1,
3761 Register t2,
3762 Label& slow_case) {
3763 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3764 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3765 }
3766
3767 // Defines obj, preserves var_size_in_bytes
3768 void MacroAssembler::eden_allocate(Register thread, Register obj,
3769 Register var_size_in_bytes,
3770 int con_size_in_bytes,
3771 Register t1,
3772 Label& slow_case) {
3773 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3774 bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
3775 }
3776
3777 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
3778 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
3779 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
3780 assert((offset_in_bytes & (BytesPerInt - 1)) == 0, "offset must be a multiple of BytesPerInt");
3781 Label done;
3782
3783 testptr(length_in_bytes, length_in_bytes);
3784 jcc(Assembler::zero, done);
3785
3786 // Emit single 32bit store to clear leading bytes, if necessary.
3787 xorptr(temp, temp); // use _zero reg to clear memory (shorter code)
3788 #ifdef _LP64
3789 if (!is_aligned(offset_in_bytes, BytesPerWord)) {
3790 movl(Address(address, offset_in_bytes), temp);
3791 offset_in_bytes += BytesPerInt;
3792 decrement(length_in_bytes, BytesPerInt);
3793 }
3794 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
3795 testptr(length_in_bytes, length_in_bytes);
3796 jcc(Assembler::zero, done);
3797 #endif
3798
3799 // initialize topmost word, divide index by 2, check if odd and test if zero
3800 // note: for the remaining code to work, index must be a multiple of BytesPerWord
3801 #ifdef ASSERT
3802 {
3803 Label L;
3804 testptr(length_in_bytes, BytesPerWord - 1);
3805 jcc(Assembler::zero, L);
3806 stop("length must be a multiple of BytesPerWord");
3807 bind(L);
3808 }
3809 #endif
3810 Register index = length_in_bytes;
3811 if (UseIncDec) {
3812 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set
3813 } else {
3814 shrptr(index, 2); // use 2 instructions to avoid partial flag stall
3815 shrptr(index, 1);
3816 }
3817 #ifndef _LP64
3818 // index could have not been a multiple of 8 (i.e., bit 2 was set)
3819 {
3820 Label even;
3821 // note: if index was a multiple of 8, then it cannot
3822 // be 0 now otherwise it must have been 0 before
3823 // => if it is even, we don't need to check for 0 again
3824 jcc(Assembler::carryClear, even);
3825 // clear topmost word (no jump would be needed if conditional assignment worked here)
3826 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
3827 // index could be 0 now, must check again
3828 jcc(Assembler::zero, done);
3829 bind(even);
3830 }
4734
4735 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
4736 // get mirror
4737 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4738 load_method_holder(mirror, method);
4739 movptr(mirror, Address(mirror, mirror_offset));
4740 resolve_oop_handle(mirror, tmp);
4741 }
4742
4743 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4744 load_method_holder(rresult, rmethod);
4745 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4746 }
4747
4748 void MacroAssembler::load_method_holder(Register holder, Register method) {
4749 movptr(holder, Address(method, Method::const_offset())); // ConstMethod*
4750 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
4751 movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4752 }
4753
4754 #ifdef _LP64
4755 void MacroAssembler::load_nklass(Register dst, Register src) {
4756 assert(UseCompressedClassPointers, "expect compressed class pointers");
4757
4758 if (!UseCompactObjectHeaders) {
4759 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4760 return;
4761 }
4762
4763 Label fast;
4764 movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
4765 testb(dst, markWord::monitor_value);
4766 jccb(Assembler::zero, fast);
4767
4768 // Fetch displaced header
4769 movq(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
4770
4771 bind(fast);
4772 shrq(dst, markWord::klass_shift);
4773 }
4774 #endif
4775
4776 void MacroAssembler::load_klass(Register dst, Register src, Register tmp, bool null_check_src) {
4777 assert_different_registers(src, tmp);
4778 assert_different_registers(dst, tmp);
4779 if (null_check_src) {
4780 if (UseCompactObjectHeaders) {
4781 null_check(src, oopDesc::mark_offset_in_bytes());
4782 } else {
4783 null_check(src, oopDesc::klass_offset_in_bytes());
4784 }
4785 }
4786 #ifdef _LP64
4787 if (UseCompressedClassPointers) {
4788 load_nklass(dst, src);
4789 decode_klass_not_null(dst, tmp);
4790 } else
4791 #endif
4792 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4793 }
4794
4795 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
4796 load_klass(dst, src, tmp);
4797 movptr(dst, Address(dst, Klass::prototype_header_offset()));
4798 }
4799
4800 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
4801 assert(!UseCompactObjectHeaders, "not with compact headers");
4802 assert_different_registers(src, tmp);
4803 assert_different_registers(dst, tmp);
4804 #ifdef _LP64
4805 if (UseCompressedClassPointers) {
4806 encode_klass_not_null(src, tmp);
4807 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4808 } else
4809 #endif
4810 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4811 }
4812
4813 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
4814 #ifdef _LP64
4815 if (UseCompactObjectHeaders) {
4816 // NOTE: We need to deal with possible ObjectMonitor in object header.
4817 // Eventually we might be able to do simple movl & cmpl like in
4818 // the CCP path below.
4819 load_nklass(tmp, obj);
4820 cmpl(klass, tmp);
4821 } else if (UseCompressedClassPointers) {
4822 cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
4823 } else
4824 #endif
4825 {
4826 cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
4827 }
4828 }
4829
4830 void MacroAssembler::cmp_klass(Register src, Register dst, Register tmp1, Register tmp2) {
4831 #ifdef _LP64
4832 if (UseCompactObjectHeaders) {
4833 // NOTE: We need to deal with possible ObjectMonitor in object header.
4834 // Eventually we might be able to do simple movl & cmpl like in
4835 // the CCP path below.
4836 assert(tmp2 != noreg, "need tmp2");
4837 assert_different_registers(src, dst, tmp1, tmp2);
4838 load_nklass(tmp1, src);
4839 load_nklass(tmp2, dst);
4840 cmpl(tmp1, tmp2);
4841 } else if (UseCompressedClassPointers) {
4842 movl(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
4843 cmpl(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
4844 } else
4845 #endif
4846 {
4847 movptr(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
4848 cmpptr(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
4849 }
4850 }
4851
4852 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
4853 Register tmp1, Register thread_tmp) {
4854 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4855 decorators = AccessInternal::decorator_fixup(decorators);
4856 bool as_raw = (decorators & AS_RAW) != 0;
4857 if (as_raw) {
4858 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4859 } else {
4860 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4861 }
4862 }
4863
4864 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
4865 Register tmp1, Register tmp2) {
4866 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4867 decorators = AccessInternal::decorator_fixup(decorators);
4868 bool as_raw = (decorators & AS_RAW) != 0;
4869 if (as_raw) {
8753
8754 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
8755
8756 #ifdef _LP64
8757 pop(r11);
8758 pop(r10);
8759 pop(r9);
8760 pop(r8);
8761 #endif
8762 pop(rcx);
8763 pop(rdx);
8764 LP64_ONLY(pop(rsi);)
8765 LP64_ONLY(pop(rdi);)
8766 if (thread != rax) {
8767 mov(thread, rax);
8768 pop(rax);
8769 }
8770 }
8771
8772 #endif // !WIN32 || _LP64
8773
8774 // Implements fast-locking.
8775 // Branches to slow upon failure to lock the object, with ZF cleared.
8776 // Falls through upon success with unspecified ZF.
8777 //
8778 // obj: the object to be locked
8779 // hdr: the (pre-loaded) header of the object, must be rax
8780 // thread: the thread which attempts to lock obj
8781 // tmp: a temporary register
8782 void MacroAssembler::fast_lock_impl(Register obj, Register hdr, Register thread, Register tmp, Label& slow) {
8783 assert(hdr == rax, "header must be in rax for cmpxchg");
8784 assert_different_registers(obj, hdr, thread, tmp);
8785
8786 // First we need to check if the lock-stack has room for pushing the object reference.
8787 // Note: we subtract 1 from the end-offset so that we can do a 'greater' comparison, instead
8788 // of 'greaterEqual' below, which readily clears the ZF. This makes C2 code a little simpler and
8789 // avoids one branch.
8790 cmpl(Address(thread, JavaThread::lock_stack_top_offset()), LockStack::end_offset() - 1);
8791 jcc(Assembler::greater, slow);
8792
8793 // Now we attempt to take the fast-lock.
8794 // Clear lock_mask bits (locked state).
8795 andptr(hdr, ~(int32_t)markWord::lock_mask_in_place);
8796 movptr(tmp, hdr);
8797 // Set unlocked_value bit.
8798 orptr(hdr, markWord::unlocked_value);
8799 lock();
8800 cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
8801 jcc(Assembler::notEqual, slow);
8802
8803 // If successful, push object to lock-stack.
8804 movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
8805 movptr(Address(thread, tmp), obj);
8806 incrementl(tmp, oopSize);
8807 movl(Address(thread, JavaThread::lock_stack_top_offset()), tmp);
8808 }
8809
8810 // Implements fast-unlocking.
8811 // Branches to slow upon failure, with ZF cleared.
8812 // Falls through upon success, with unspecified ZF.
8813 //
8814 // obj: the object to be unlocked
8815 // hdr: the (pre-loaded) header of the object, must be rax
8816 // tmp: a temporary register
8817 void MacroAssembler::fast_unlock_impl(Register obj, Register hdr, Register tmp, Label& slow) {
8818 assert(hdr == rax, "header must be in rax for cmpxchg");
8819 assert_different_registers(obj, hdr, tmp);
8820
8821 // Mark-word must be lock_mask now, try to swing it back to unlocked_value.
8822 movptr(tmp, hdr); // The expected old value
8823 orptr(tmp, markWord::unlocked_value);
8824 lock();
8825 cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
8826 jcc(Assembler::notEqual, slow);
8827 // Pop the lock object from the lock-stack.
8828 #ifdef _LP64
8829 const Register thread = r15_thread;
8830 #else
8831 const Register thread = rax;
8832 get_thread(thread);
8833 #endif
8834 subl(Address(thread, JavaThread::lock_stack_top_offset()), oopSize);
8835 #ifdef ASSERT
8836 movl(tmp, Address(thread, JavaThread::lock_stack_top_offset()));
8837 movptr(Address(thread, tmp), 0);
8838 #endif
8839 }
|