37 #include "memory/resourceArea.hpp"
38 #include "memory/universe.hpp"
39 #include "oops/accessDecorators.hpp"
40 #include "oops/compressedOops.inline.hpp"
41 #include "oops/klass.inline.hpp"
42 #include "prims/methodHandles.hpp"
43 #include "runtime/biasedLocking.hpp"
44 #include "runtime/flags/flagSetting.hpp"
45 #include "runtime/interfaceSupport.inline.hpp"
46 #include "runtime/jniHandles.hpp"
47 #include "runtime/objectMonitor.hpp"
48 #include "runtime/os.hpp"
49 #include "runtime/safepoint.hpp"
50 #include "runtime/safepointMechanism.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubRoutines.hpp"
53 #include "runtime/thread.hpp"
54 #include "utilities/macros.hpp"
55 #include "crc32c.h"
56
57 #ifdef PRODUCT
58 #define BLOCK_COMMENT(str) /* nothing */
59 #define STOP(error) stop(error)
60 #else
61 #define BLOCK_COMMENT(str) block_comment(str)
62 #define STOP(error) block_comment(error); stop(error)
63 #endif
64
65 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
66
67 #ifdef ASSERT
68 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
69 #endif
70
71 static Assembler::Condition reverse[] = {
72 Assembler::noOverflow /* overflow = 0x0 */ ,
73 Assembler::overflow /* noOverflow = 0x1 */ ,
74 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ ,
75 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ ,
76 Assembler::notZero /* zero = 0x4, equal = 0x4 */ ,
3754 Register t1,
3755 Register t2,
3756 Label& slow_case) {
3757 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3758 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3759 }
3760
3761 // Defines obj, preserves var_size_in_bytes
3762 void MacroAssembler::eden_allocate(Register thread, Register obj,
3763 Register var_size_in_bytes,
3764 int con_size_in_bytes,
3765 Register t1,
3766 Label& slow_case) {
3767 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3768 bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
3769 }
3770
3771 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
3772 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
3773 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
3774 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
3775 Label done;
3776
3777 testptr(length_in_bytes, length_in_bytes);
3778 jcc(Assembler::zero, done);
3779
3780 // initialize topmost word, divide index by 2, check if odd and test if zero
3781 // note: for the remaining code to work, index must be a multiple of BytesPerWord
3782 #ifdef ASSERT
3783 {
3784 Label L;
3785 testptr(length_in_bytes, BytesPerWord - 1);
3786 jcc(Assembler::zero, L);
3787 stop("length must be a multiple of BytesPerWord");
3788 bind(L);
3789 }
3790 #endif
3791 Register index = length_in_bytes;
3792 xorptr(temp, temp); // use _zero reg to clear memory (shorter code)
3793 if (UseIncDec) {
3794 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set
3795 } else {
3796 shrptr(index, 2); // use 2 instructions to avoid partial flag stall
3797 shrptr(index, 1);
3798 }
3799 #ifndef _LP64
3800 // index could have not been a multiple of 8 (i.e., bit 2 was set)
3801 {
3802 Label even;
3803 // note: if index was a multiple of 8, then it cannot
3804 // be 0 now otherwise it must have been 0 before
3805 // => if it is even, we don't need to check for 0 again
3806 jcc(Assembler::carryClear, even);
3807 // clear topmost word (no jump would be needed if conditional assignment worked here)
3808 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
3809 // index could be 0 now, must check again
3810 jcc(Assembler::zero, done);
3811 bind(even);
3812 }
4716
4717 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
4718 // get mirror
4719 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4720 load_method_holder(mirror, method);
4721 movptr(mirror, Address(mirror, mirror_offset));
4722 resolve_oop_handle(mirror, tmp);
4723 }
4724
4725 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4726 load_method_holder(rresult, rmethod);
4727 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4728 }
4729
4730 void MacroAssembler::load_method_holder(Register holder, Register method) {
4731 movptr(holder, Address(method, Method::const_offset())); // ConstMethod*
4732 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
4733 movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4734 }
4735
4736 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
4737 assert_different_registers(src, tmp);
4738 assert_different_registers(dst, tmp);
4739 #ifdef _LP64
4740 if (UseCompressedClassPointers) {
4741 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4742 decode_klass_not_null(dst, tmp);
4743 } else
4744 #endif
4745 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4746 }
4747
4748 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
4749 load_klass(dst, src, tmp);
4750 movptr(dst, Address(dst, Klass::prototype_header_offset()));
4751 }
4752
4753 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
4754 assert_different_registers(src, tmp);
4755 assert_different_registers(dst, tmp);
4756 #ifdef _LP64
4757 if (UseCompressedClassPointers) {
4758 encode_klass_not_null(src, tmp);
4759 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4760 } else
4761 #endif
4762 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4763 }
4764
4765 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
4766 Register tmp1, Register thread_tmp) {
4767 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4768 decorators = AccessInternal::decorator_fixup(decorators);
4769 bool as_raw = (decorators & AS_RAW) != 0;
4770 if (as_raw) {
4771 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4772 } else {
4773 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4774 }
4775 }
4776
4777 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
4778 Register tmp1, Register tmp2) {
4779 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4780 decorators = AccessInternal::decorator_fixup(decorators);
4781 bool as_raw = (decorators & AS_RAW) != 0;
4782 if (as_raw) {
5102 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5103 }
5104
5105 void MacroAssembler::reinit_heapbase() {
5106 if (UseCompressedOops) {
5107 if (Universe::heap() != NULL) {
5108 if (CompressedOops::base() == NULL) {
5109 MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
5110 } else {
5111 mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
5112 }
5113 } else {
5114 movptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
5115 }
5116 }
5117 }
5118
5119 #endif // _LP64
5120
5121 // C2 compiled method's prolog code.
5122 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub) {
5123
5124 // WARNING: Initial instruction MUST be 5 bytes or longer so that
5125 // NativeJump::patch_verified_entry will be able to patch out the entry
5126 // code safely. The push to verify stack depth is ok at 5 bytes,
5127 // the frame allocation can be either 3 or 6 bytes. So if we don't do
5128 // stack bang then we must use the 6 byte frame allocation even if
5129 // we have no frame. :-(
5130 assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
5131
5132 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
5133 // Remove word for return addr
5134 framesize -= wordSize;
5135 stack_bang_size -= wordSize;
5136
5137 // Calls to C2R adapters often do not accept exceptional returns.
5138 // We require that their callers must bang for them. But be careful, because
5139 // some VM calls (such as call site linkage) can use several kilobytes of
5140 // stack. But the stack safety zone should account for that.
5141 // See bugs 4446381, 4468289, 4497237.
5142 if (stack_bang_size > 0) {
5184 }
5185 if (UseSSE >= 2 && VerifyFPU) {
5186 verify_FPU(0, "FPU stack must be clean on entry");
5187 }
5188 #endif
5189
5190 #ifdef ASSERT
5191 if (VerifyStackAtCalls) {
5192 Label L;
5193 push(rax);
5194 mov(rax, rsp);
5195 andptr(rax, StackAlignmentInBytes-1);
5196 cmpptr(rax, StackAlignmentInBytes-wordSize);
5197 pop(rax);
5198 jcc(Assembler::equal, L);
5199 STOP("Stack is not properly aligned!");
5200 bind(L);
5201 }
5202 #endif
5203
5204 if (!is_stub) {
5205 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5206 bs->nmethod_entry_barrier(this);
5207 }
5208 }
5209
5210 #if COMPILER2_OR_JVMCI
5211
5212 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
5213 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5214 // cnt - number of qwords (8-byte words).
5215 // base - start address, qword aligned.
5216 Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
5217 bool use64byteVector = MaxVectorSize == 64 && AVX3Threshold == 0;
5218 if (use64byteVector) {
5219 vpxor(xtmp, xtmp, xtmp, AVX_512bit);
5220 } else if (MaxVectorSize >= 32) {
5221 vpxor(xtmp, xtmp, xtmp, AVX_256bit);
5222 } else {
5223 pxor(xtmp, xtmp);
8666
8667 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
8668
8669 #ifdef _LP64
8670 pop(r11);
8671 pop(r10);
8672 pop(r9);
8673 pop(r8);
8674 #endif
8675 pop(rcx);
8676 pop(rdx);
8677 LP64_ONLY(pop(rsi);)
8678 LP64_ONLY(pop(rdi);)
8679 if (thread != rax) {
8680 mov(thread, rax);
8681 pop(rax);
8682 }
8683 }
8684
8685 #endif // !WIN32 || _LP64
|
37 #include "memory/resourceArea.hpp"
38 #include "memory/universe.hpp"
39 #include "oops/accessDecorators.hpp"
40 #include "oops/compressedOops.inline.hpp"
41 #include "oops/klass.inline.hpp"
42 #include "prims/methodHandles.hpp"
43 #include "runtime/biasedLocking.hpp"
44 #include "runtime/flags/flagSetting.hpp"
45 #include "runtime/interfaceSupport.inline.hpp"
46 #include "runtime/jniHandles.hpp"
47 #include "runtime/objectMonitor.hpp"
48 #include "runtime/os.hpp"
49 #include "runtime/safepoint.hpp"
50 #include "runtime/safepointMechanism.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubRoutines.hpp"
53 #include "runtime/thread.hpp"
54 #include "utilities/macros.hpp"
55 #include "crc32c.h"
56
57 #ifdef COMPILER2
58 #include "opto/c2_CodeStubs.hpp"
59 #include "opto/compile.hpp"
60 #include "opto/output.hpp"
61 #endif
62
63 #ifdef PRODUCT
64 #define BLOCK_COMMENT(str) /* nothing */
65 #define STOP(error) stop(error)
66 #else
67 #define BLOCK_COMMENT(str) block_comment(str)
68 #define STOP(error) block_comment(error); stop(error)
69 #endif
70
71 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
72
73 #ifdef ASSERT
74 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
75 #endif
76
77 static Assembler::Condition reverse[] = {
78 Assembler::noOverflow /* overflow = 0x0 */ ,
79 Assembler::overflow /* noOverflow = 0x1 */ ,
80 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ ,
81 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ ,
82 Assembler::notZero /* zero = 0x4, equal = 0x4 */ ,
3760 Register t1,
3761 Register t2,
3762 Label& slow_case) {
3763 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3764 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3765 }
3766
3767 // Defines obj, preserves var_size_in_bytes
3768 void MacroAssembler::eden_allocate(Register thread, Register obj,
3769 Register var_size_in_bytes,
3770 int con_size_in_bytes,
3771 Register t1,
3772 Label& slow_case) {
3773 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3774 bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
3775 }
3776
3777 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
3778 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
3779 assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
3780 assert((offset_in_bytes & (BytesPerInt - 1)) == 0, "offset must be a multiple of BytesPerInt");
3781 Label done;
3782
3783 testptr(length_in_bytes, length_in_bytes);
3784 jcc(Assembler::zero, done);
3785
3786 // Emit single 32bit store to clear leading bytes, if necessary.
3787 xorptr(temp, temp); // use _zero reg to clear memory (shorter code)
3788 #ifdef _LP64
3789 if (!is_aligned(offset_in_bytes, BytesPerWord)) {
3790 movl(Address(address, offset_in_bytes), temp);
3791 offset_in_bytes += BytesPerInt;
3792 decrement(length_in_bytes, BytesPerInt);
3793 }
3794 assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
3795 testptr(length_in_bytes, length_in_bytes);
3796 jcc(Assembler::zero, done);
3797 #endif
3798
3799 // initialize topmost word, divide index by 2, check if odd and test if zero
3800 // note: for the remaining code to work, index must be a multiple of BytesPerWord
3801 #ifdef ASSERT
3802 {
3803 Label L;
3804 testptr(length_in_bytes, BytesPerWord - 1);
3805 jcc(Assembler::zero, L);
3806 stop("length must be a multiple of BytesPerWord");
3807 bind(L);
3808 }
3809 #endif
3810 Register index = length_in_bytes;
3811 if (UseIncDec) {
3812 shrptr(index, 3); // divide by 8/16 and set carry flag if bit 2 was set
3813 } else {
3814 shrptr(index, 2); // use 2 instructions to avoid partial flag stall
3815 shrptr(index, 1);
3816 }
3817 #ifndef _LP64
3818 // index could have not been a multiple of 8 (i.e., bit 2 was set)
3819 {
3820 Label even;
3821 // note: if index was a multiple of 8, then it cannot
3822 // be 0 now otherwise it must have been 0 before
3823 // => if it is even, we don't need to check for 0 again
3824 jcc(Assembler::carryClear, even);
3825 // clear topmost word (no jump would be needed if conditional assignment worked here)
3826 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
3827 // index could be 0 now, must check again
3828 jcc(Assembler::zero, done);
3829 bind(even);
3830 }
4734
4735 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
4736 // get mirror
4737 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4738 load_method_holder(mirror, method);
4739 movptr(mirror, Address(mirror, mirror_offset));
4740 resolve_oop_handle(mirror, tmp);
4741 }
4742
4743 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4744 load_method_holder(rresult, rmethod);
4745 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4746 }
4747
4748 void MacroAssembler::load_method_holder(Register holder, Register method) {
4749 movptr(holder, Address(method, Method::const_offset())); // ConstMethod*
4750 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
4751 movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4752 }
4753
4754 #ifdef _LP64
4755 void MacroAssembler::load_nklass(Register dst, Register src) {
4756 assert(UseCompressedClassPointers, "expect compressed class pointers");
4757
4758 if (!UseCompactObjectHeaders) {
4759 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4760 return;
4761 }
4762
4763 Label fast;
4764 movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
4765 testb(dst, markWord::monitor_value);
4766 jccb(Assembler::zero, fast);
4767
4768 // Fetch displaced header
4769 movq(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
4770
4771 bind(fast);
4772 shrq(dst, markWord::klass_shift);
4773 }
4774 #endif
4775
4776 void MacroAssembler::load_klass(Register dst, Register src, Register tmp, bool null_check_src) {
4777 assert_different_registers(src, tmp);
4778 assert_different_registers(dst, tmp);
4779 if (null_check_src) {
4780 if (UseCompactObjectHeaders) {
4781 null_check(src, oopDesc::mark_offset_in_bytes());
4782 } else {
4783 null_check(src, oopDesc::klass_offset_in_bytes());
4784 }
4785 }
4786 #ifdef _LP64
4787 if (UseCompressedClassPointers) {
4788 load_nklass(dst, src);
4789 decode_klass_not_null(dst, tmp);
4790 } else
4791 #endif
4792 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4793 }
4794
4795 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
4796 load_klass(dst, src, tmp);
4797 movptr(dst, Address(dst, Klass::prototype_header_offset()));
4798 }
4799
4800 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
4801 assert(!UseCompactObjectHeaders, "not with compact headers");
4802 assert_different_registers(src, tmp);
4803 assert_different_registers(dst, tmp);
4804 #ifdef _LP64
4805 if (UseCompressedClassPointers) {
4806 encode_klass_not_null(src, tmp);
4807 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4808 } else
4809 #endif
4810 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4811 }
4812
4813 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
4814 #ifdef _LP64
4815 if (UseCompactObjectHeaders) {
4816 // NOTE: We need to deal with possible ObjectMonitor in object header.
4817 // Eventually we might be able to do simple movl & cmpl like in
4818 // the CCP path below.
4819 load_nklass(tmp, obj);
4820 cmpl(klass, tmp);
4821 } else if (UseCompressedClassPointers) {
4822 cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
4823 } else
4824 #endif
4825 {
4826 cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
4827 }
4828 }
4829
4830 void MacroAssembler::cmp_klass(Register src, Register dst, Register tmp1, Register tmp2) {
4831 #ifdef _LP64
4832 if (UseCompactObjectHeaders) {
4833 // NOTE: We need to deal with possible ObjectMonitor in object header.
4834 // Eventually we might be able to do simple movl & cmpl like in
4835 // the CCP path below.
4836 assert(tmp2 != noreg, "need tmp2");
4837 assert_different_registers(src, dst, tmp1, tmp2);
4838 load_nklass(tmp1, src);
4839 load_nklass(tmp2, dst);
4840 cmpl(tmp1, tmp2);
4841 } else if (UseCompressedClassPointers) {
4842 movl(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
4843 cmpl(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
4844 } else
4845 #endif
4846 {
4847 movptr(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
4848 cmpptr(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
4849 }
4850 }
4851
4852 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
4853 Register tmp1, Register thread_tmp) {
4854 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4855 decorators = AccessInternal::decorator_fixup(decorators);
4856 bool as_raw = (decorators & AS_RAW) != 0;
4857 if (as_raw) {
4858 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4859 } else {
4860 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4861 }
4862 }
4863
4864 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
4865 Register tmp1, Register tmp2) {
4866 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4867 decorators = AccessInternal::decorator_fixup(decorators);
4868 bool as_raw = (decorators & AS_RAW) != 0;
4869 if (as_raw) {
5189 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5190 }
5191
5192 void MacroAssembler::reinit_heapbase() {
5193 if (UseCompressedOops) {
5194 if (Universe::heap() != NULL) {
5195 if (CompressedOops::base() == NULL) {
5196 MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
5197 } else {
5198 mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
5199 }
5200 } else {
5201 movptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
5202 }
5203 }
5204 }
5205
5206 #endif // _LP64
5207
5208 // C2 compiled method's prolog code.
5209 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub, int max_monitors) {
5210
5211 // WARNING: Initial instruction MUST be 5 bytes or longer so that
5212 // NativeJump::patch_verified_entry will be able to patch out the entry
5213 // code safely. The push to verify stack depth is ok at 5 bytes,
5214 // the frame allocation can be either 3 or 6 bytes. So if we don't do
5215 // stack bang then we must use the 6 byte frame allocation even if
5216 // we have no frame. :-(
5217 assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
5218
5219 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
5220 // Remove word for return addr
5221 framesize -= wordSize;
5222 stack_bang_size -= wordSize;
5223
5224 // Calls to C2R adapters often do not accept exceptional returns.
5225 // We require that their callers must bang for them. But be careful, because
5226 // some VM calls (such as call site linkage) can use several kilobytes of
5227 // stack. But the stack safety zone should account for that.
5228 // See bugs 4446381, 4468289, 4497237.
5229 if (stack_bang_size > 0) {
5271 }
5272 if (UseSSE >= 2 && VerifyFPU) {
5273 verify_FPU(0, "FPU stack must be clean on entry");
5274 }
5275 #endif
5276
5277 #ifdef ASSERT
5278 if (VerifyStackAtCalls) {
5279 Label L;
5280 push(rax);
5281 mov(rax, rsp);
5282 andptr(rax, StackAlignmentInBytes-1);
5283 cmpptr(rax, StackAlignmentInBytes-wordSize);
5284 pop(rax);
5285 jcc(Assembler::equal, L);
5286 STOP("Stack is not properly aligned!");
5287 bind(L);
5288 }
5289 #endif
5290
5291 #if defined(_LP64) && defined(COMPILER2)
5292 if (UseFastLocking && max_monitors > 0) {
5293 C2CheckLockStackStub* stub = new (Compile::current()->comp_arena()) C2CheckLockStackStub();
5294 Compile::current()->output()->add_stub(stub);
5295 assert(!is_stub, "only methods have monitors");
5296 Register thread = r15_thread;
5297 movptr(rax, Address(thread, JavaThread::lock_stack_current_offset()));
5298 addptr(rax, max_monitors * wordSize);
5299 cmpptr(rax, Address(thread, JavaThread::lock_stack_limit_offset()));
5300 jcc(Assembler::greaterEqual, stub->entry());
5301 bind(stub->continuation());
5302 }
5303 #endif
5304
5305 if (!is_stub) {
5306 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5307 bs->nmethod_entry_barrier(this);
5308 }
5309 }
5310
5311 #if COMPILER2_OR_JVMCI
5312
5313 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
5314 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5315 // cnt - number of qwords (8-byte words).
5316 // base - start address, qword aligned.
5317 Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
5318 bool use64byteVector = MaxVectorSize == 64 && AVX3Threshold == 0;
5319 if (use64byteVector) {
5320 vpxor(xtmp, xtmp, xtmp, AVX_512bit);
5321 } else if (MaxVectorSize >= 32) {
5322 vpxor(xtmp, xtmp, xtmp, AVX_256bit);
5323 } else {
5324 pxor(xtmp, xtmp);
8767
8768 MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
8769
8770 #ifdef _LP64
8771 pop(r11);
8772 pop(r10);
8773 pop(r9);
8774 pop(r8);
8775 #endif
8776 pop(rcx);
8777 pop(rdx);
8778 LP64_ONLY(pop(rsi);)
8779 LP64_ONLY(pop(rdi);)
8780 if (thread != rax) {
8781 mov(thread, rax);
8782 pop(rax);
8783 }
8784 }
8785
8786 #endif // !WIN32 || _LP64
8787
8788 void MacroAssembler::fast_lock_impl(Register obj, Register hdr, Register thread, Register tmp, Label& slow, bool rt_check_stack) {
8789 assert(hdr == rax, "header must be in rax for cmpxchg");
8790 assert_different_registers(obj, hdr, thread, tmp);
8791
8792 // First we need to check if the lock-stack has room for pushing the object reference.
8793 if (rt_check_stack) {
8794 movptr(tmp, Address(thread, JavaThread::lock_stack_current_offset()));
8795 cmpptr(tmp, Address(thread, JavaThread::lock_stack_limit_offset()));
8796 jcc(Assembler::greaterEqual, slow);
8797 }
8798 #ifdef ASSERT
8799 else {
8800 Label ok;
8801 movptr(tmp, Address(thread, JavaThread::lock_stack_current_offset()));
8802 cmpptr(tmp, Address(thread, JavaThread::lock_stack_limit_offset()));
8803 jcc(Assembler::less, ok);
8804 stop("Not enough room in lock stack; should have been checked in the method prologue");
8805 bind(ok);
8806 }
8807 #endif
8808
8809 // Now we attempt to take the fast-lock.
8810 // Clear lowest two header bits (locked state).
8811 andptr(hdr, ~(int32_t )markWord::lock_mask_in_place);
8812 movptr(tmp, hdr);
8813 // Set lowest bit (unlocked state).
8814 orptr(hdr, markWord::unlocked_value);
8815 lock();
8816 cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
8817 jcc(Assembler::notEqual, slow);
8818
8819 // If successful, push object to lock-stack.
8820 movptr(tmp, Address(thread, JavaThread::lock_stack_current_offset()));
8821 movptr(Address(tmp, 0), obj);
8822 addptr(tmp, oopSize);
8823 movptr(Address(thread, JavaThread::lock_stack_current_offset()), tmp);
8824 }
8825
8826 void MacroAssembler::fast_unlock_impl(Register obj, Register hdr, Register tmp, Label& slow) {
8827 assert(hdr == rax, "header must be in rax for cmpxchg");
8828 assert_different_registers(obj, hdr, tmp);
8829
8830 // Mark-word must be 00 now, try to swing it back to 01 (unlocked)
8831 movptr(tmp, hdr); // The expected old value
8832 orptr(tmp, markWord::unlocked_value);
8833 lock();
8834 cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
8835 jcc(Assembler::notEqual, slow);
8836 // Pop the lock object from the lock-stack.
8837 #ifdef _LP64
8838 const Register thread = r15_thread;
8839 #else
8840 const Register thread = rax;
8841 get_thread(rax);
8842 #endif
8843 subptr(Address(thread, JavaThread::lock_stack_current_offset()), oopSize);
8844 }
|