< prev index next >

src/hotspot/cpu/x86/macroAssembler_x86.cpp

Print this page

  37 #include "memory/resourceArea.hpp"
  38 #include "memory/universe.hpp"
  39 #include "oops/accessDecorators.hpp"
  40 #include "oops/compressedOops.inline.hpp"
  41 #include "oops/klass.inline.hpp"
  42 #include "prims/methodHandles.hpp"
  43 #include "runtime/biasedLocking.hpp"
  44 #include "runtime/flags/flagSetting.hpp"
  45 #include "runtime/interfaceSupport.inline.hpp"
  46 #include "runtime/jniHandles.hpp"
  47 #include "runtime/objectMonitor.hpp"
  48 #include "runtime/os.hpp"
  49 #include "runtime/safepoint.hpp"
  50 #include "runtime/safepointMechanism.hpp"
  51 #include "runtime/sharedRuntime.hpp"
  52 #include "runtime/stubRoutines.hpp"
  53 #include "runtime/thread.hpp"
  54 #include "utilities/macros.hpp"
  55 #include "crc32c.h"
  56 






  57 #ifdef PRODUCT
  58 #define BLOCK_COMMENT(str) /* nothing */
  59 #define STOP(error) stop(error)
  60 #else
  61 #define BLOCK_COMMENT(str) block_comment(str)
  62 #define STOP(error) block_comment(error); stop(error)
  63 #endif
  64 
  65 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  66 
  67 #ifdef ASSERT
  68 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  69 #endif
  70 
  71 static Assembler::Condition reverse[] = {
  72     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  73     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  74     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  75     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  76     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,

3754                                    Register t1,
3755                                    Register t2,
3756                                    Label& slow_case) {
3757   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3758   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3759 }
3760 
3761 // Defines obj, preserves var_size_in_bytes
3762 void MacroAssembler::eden_allocate(Register thread, Register obj,
3763                                    Register var_size_in_bytes,
3764                                    int con_size_in_bytes,
3765                                    Register t1,
3766                                    Label& slow_case) {
3767   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3768   bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
3769 }
3770 
3771 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
3772 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
3773   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
3774   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
3775   Label done;
3776 
3777   testptr(length_in_bytes, length_in_bytes);
3778   jcc(Assembler::zero, done);
3779 













3780   // initialize topmost word, divide index by 2, check if odd and test if zero
3781   // note: for the remaining code to work, index must be a multiple of BytesPerWord
3782 #ifdef ASSERT
3783   {
3784     Label L;
3785     testptr(length_in_bytes, BytesPerWord - 1);
3786     jcc(Assembler::zero, L);
3787     stop("length must be a multiple of BytesPerWord");
3788     bind(L);
3789   }
3790 #endif
3791   Register index = length_in_bytes;
3792   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
3793   if (UseIncDec) {
3794     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
3795   } else {
3796     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
3797     shrptr(index, 1);
3798   }
3799 #ifndef _LP64
3800   // index could have not been a multiple of 8 (i.e., bit 2 was set)
3801   {
3802     Label even;
3803     // note: if index was a multiple of 8, then it cannot
3804     //       be 0 now otherwise it must have been 0 before
3805     //       => if it is even, we don't need to check for 0 again
3806     jcc(Assembler::carryClear, even);
3807     // clear topmost word (no jump would be needed if conditional assignment worked here)
3808     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
3809     // index could be 0 now, must check again
3810     jcc(Assembler::zero, done);
3811     bind(even);
3812   }

4716 
4717 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
4718   // get mirror
4719   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4720   load_method_holder(mirror, method);
4721   movptr(mirror, Address(mirror, mirror_offset));
4722   resolve_oop_handle(mirror, tmp);
4723 }
4724 
4725 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4726   load_method_holder(rresult, rmethod);
4727   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4728 }
4729 
4730 void MacroAssembler::load_method_holder(Register holder, Register method) {
4731   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
4732   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
4733   movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4734 }
4735 
4736 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {






















4737   assert_different_registers(src, tmp);
4738   assert_different_registers(dst, tmp);







4739 #ifdef _LP64
4740   if (UseCompressedClassPointers) {
4741     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4742     decode_klass_not_null(dst, tmp);
4743   } else
4744 #endif
4745     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4746 }
4747 
4748 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
4749   load_klass(dst, src, tmp);
4750   movptr(dst, Address(dst, Klass::prototype_header_offset()));
4751 }
4752 
4753 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {

4754   assert_different_registers(src, tmp);
4755   assert_different_registers(dst, tmp);
4756 #ifdef _LP64
4757   if (UseCompressedClassPointers) {
4758     encode_klass_not_null(src, tmp);
4759     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4760   } else
4761 #endif
4762     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);







































4763 }
4764 
4765 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
4766                                     Register tmp1, Register thread_tmp) {
4767   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4768   decorators = AccessInternal::decorator_fixup(decorators);
4769   bool as_raw = (decorators & AS_RAW) != 0;
4770   if (as_raw) {
4771     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4772   } else {
4773     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4774   }
4775 }
4776 
4777 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
4778                                      Register tmp1, Register tmp2) {
4779   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4780   decorators = AccessInternal::decorator_fixup(decorators);
4781   bool as_raw = (decorators & AS_RAW) != 0;
4782   if (as_raw) {

5102   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5103 }
5104 
5105 void MacroAssembler::reinit_heapbase() {
5106   if (UseCompressedOops) {
5107     if (Universe::heap() != NULL) {
5108       if (CompressedOops::base() == NULL) {
5109         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
5110       } else {
5111         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
5112       }
5113     } else {
5114       movptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
5115     }
5116   }
5117 }
5118 
5119 #endif // _LP64
5120 
5121 // C2 compiled method's prolog code.
5122 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub) {
5123 
5124   // WARNING: Initial instruction MUST be 5 bytes or longer so that
5125   // NativeJump::patch_verified_entry will be able to patch out the entry
5126   // code safely. The push to verify stack depth is ok at 5 bytes,
5127   // the frame allocation can be either 3 or 6 bytes. So if we don't do
5128   // stack bang then we must use the 6 byte frame allocation even if
5129   // we have no frame. :-(
5130   assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
5131 
5132   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
5133   // Remove word for return addr
5134   framesize -= wordSize;
5135   stack_bang_size -= wordSize;
5136 
5137   // Calls to C2R adapters often do not accept exceptional returns.
5138   // We require that their callers must bang for them.  But be careful, because
5139   // some VM calls (such as call site linkage) can use several kilobytes of
5140   // stack.  But the stack safety zone should account for that.
5141   // See bugs 4446381, 4468289, 4497237.
5142   if (stack_bang_size > 0) {

5184   }
5185   if (UseSSE >= 2 && VerifyFPU) {
5186     verify_FPU(0, "FPU stack must be clean on entry");
5187   }
5188 #endif
5189 
5190 #ifdef ASSERT
5191   if (VerifyStackAtCalls) {
5192     Label L;
5193     push(rax);
5194     mov(rax, rsp);
5195     andptr(rax, StackAlignmentInBytes-1);
5196     cmpptr(rax, StackAlignmentInBytes-wordSize);
5197     pop(rax);
5198     jcc(Assembler::equal, L);
5199     STOP("Stack is not properly aligned!");
5200     bind(L);
5201   }
5202 #endif
5203 














5204   if (!is_stub) {
5205     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5206     bs->nmethod_entry_barrier(this);
5207   }
5208 }
5209 
5210 #if COMPILER2_OR_JVMCI
5211 
5212 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
5213 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5214   // cnt - number of qwords (8-byte words).
5215   // base - start address, qword aligned.
5216   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
5217   bool use64byteVector = MaxVectorSize == 64 && AVX3Threshold == 0;
5218   if (use64byteVector) {
5219     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
5220   } else if (MaxVectorSize >= 32) {
5221     vpxor(xtmp, xtmp, xtmp, AVX_256bit);
5222   } else {
5223     pxor(xtmp, xtmp);

8666 
8667   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
8668 
8669 #ifdef _LP64
8670   pop(r11);
8671   pop(r10);
8672   pop(r9);
8673   pop(r8);
8674 #endif
8675   pop(rcx);
8676   pop(rdx);
8677   LP64_ONLY(pop(rsi);)
8678   LP64_ONLY(pop(rdi);)
8679   if (thread != rax) {
8680     mov(thread, rax);
8681     pop(rax);
8682   }
8683 }
8684 
8685 #endif // !WIN32 || _LP64



























































  37 #include "memory/resourceArea.hpp"
  38 #include "memory/universe.hpp"
  39 #include "oops/accessDecorators.hpp"
  40 #include "oops/compressedOops.inline.hpp"
  41 #include "oops/klass.inline.hpp"
  42 #include "prims/methodHandles.hpp"
  43 #include "runtime/biasedLocking.hpp"
  44 #include "runtime/flags/flagSetting.hpp"
  45 #include "runtime/interfaceSupport.inline.hpp"
  46 #include "runtime/jniHandles.hpp"
  47 #include "runtime/objectMonitor.hpp"
  48 #include "runtime/os.hpp"
  49 #include "runtime/safepoint.hpp"
  50 #include "runtime/safepointMechanism.hpp"
  51 #include "runtime/sharedRuntime.hpp"
  52 #include "runtime/stubRoutines.hpp"
  53 #include "runtime/thread.hpp"
  54 #include "utilities/macros.hpp"
  55 #include "crc32c.h"
  56 
  57 #ifdef COMPILER2
  58 #include "opto/c2_CodeStubs.hpp"
  59 #include "opto/compile.hpp"
  60 #include "opto/output.hpp"
  61 #endif
  62 
  63 #ifdef PRODUCT
  64 #define BLOCK_COMMENT(str) /* nothing */
  65 #define STOP(error) stop(error)
  66 #else
  67 #define BLOCK_COMMENT(str) block_comment(str)
  68 #define STOP(error) block_comment(error); stop(error)
  69 #endif
  70 
  71 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  72 
  73 #ifdef ASSERT
  74 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
  75 #endif
  76 
  77 static Assembler::Condition reverse[] = {
  78     Assembler::noOverflow     /* overflow      = 0x0 */ ,
  79     Assembler::overflow       /* noOverflow    = 0x1 */ ,
  80     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
  81     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,
  82     Assembler::notZero        /* zero          = 0x4, equal         = 0x4 */ ,

3760                                    Register t1,
3761                                    Register t2,
3762                                    Label& slow_case) {
3763   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3764   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3765 }
3766 
3767 // Defines obj, preserves var_size_in_bytes
3768 void MacroAssembler::eden_allocate(Register thread, Register obj,
3769                                    Register var_size_in_bytes,
3770                                    int con_size_in_bytes,
3771                                    Register t1,
3772                                    Label& slow_case) {
3773   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3774   bs->eden_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
3775 }
3776 
3777 // Preserves the contents of address, destroys the contents length_in_bytes and temp.
3778 void MacroAssembler::zero_memory(Register address, Register length_in_bytes, int offset_in_bytes, Register temp) {
3779   assert(address != length_in_bytes && address != temp && temp != length_in_bytes, "registers must be different");
3780   assert((offset_in_bytes & (BytesPerInt - 1)) == 0, "offset must be a multiple of BytesPerInt");
3781   Label done;
3782 
3783   testptr(length_in_bytes, length_in_bytes);
3784   jcc(Assembler::zero, done);
3785 
3786   // Emit single 32bit store to clear leading bytes, if necessary.
3787   xorptr(temp, temp);    // use _zero reg to clear memory (shorter code)
3788 #ifdef _LP64
3789   if (!is_aligned(offset_in_bytes, BytesPerWord)) {
3790     movl(Address(address, offset_in_bytes), temp);
3791     offset_in_bytes += BytesPerInt;
3792     decrement(length_in_bytes, BytesPerInt);
3793   }
3794   assert((offset_in_bytes & (BytesPerWord - 1)) == 0, "offset must be a multiple of BytesPerWord");
3795   testptr(length_in_bytes, length_in_bytes);
3796   jcc(Assembler::zero, done);
3797 #endif
3798 
3799   // initialize topmost word, divide index by 2, check if odd and test if zero
3800   // note: for the remaining code to work, index must be a multiple of BytesPerWord
3801 #ifdef ASSERT
3802   {
3803     Label L;
3804     testptr(length_in_bytes, BytesPerWord - 1);
3805     jcc(Assembler::zero, L);
3806     stop("length must be a multiple of BytesPerWord");
3807     bind(L);
3808   }
3809 #endif
3810   Register index = length_in_bytes;

3811   if (UseIncDec) {
3812     shrptr(index, 3);  // divide by 8/16 and set carry flag if bit 2 was set
3813   } else {
3814     shrptr(index, 2);  // use 2 instructions to avoid partial flag stall
3815     shrptr(index, 1);
3816   }
3817 #ifndef _LP64
3818   // index could have not been a multiple of 8 (i.e., bit 2 was set)
3819   {
3820     Label even;
3821     // note: if index was a multiple of 8, then it cannot
3822     //       be 0 now otherwise it must have been 0 before
3823     //       => if it is even, we don't need to check for 0 again
3824     jcc(Assembler::carryClear, even);
3825     // clear topmost word (no jump would be needed if conditional assignment worked here)
3826     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
3827     // index could be 0 now, must check again
3828     jcc(Assembler::zero, done);
3829     bind(even);
3830   }

4734 
4735 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
4736   // get mirror
4737   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4738   load_method_holder(mirror, method);
4739   movptr(mirror, Address(mirror, mirror_offset));
4740   resolve_oop_handle(mirror, tmp);
4741 }
4742 
4743 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4744   load_method_holder(rresult, rmethod);
4745   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4746 }
4747 
4748 void MacroAssembler::load_method_holder(Register holder, Register method) {
4749   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
4750   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
4751   movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4752 }
4753 
4754 #ifdef _LP64
4755 void MacroAssembler::load_nklass(Register dst, Register src) {
4756   assert(UseCompressedClassPointers, "expect compressed class pointers");
4757 
4758   if (!UseCompactObjectHeaders) {
4759     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4760     return;
4761   }
4762 
4763  Label fast;
4764   movq(dst, Address(src, oopDesc::mark_offset_in_bytes()));
4765   testb(dst, markWord::monitor_value);
4766   jccb(Assembler::zero, fast);
4767 
4768   // Fetch displaced header
4769   movq(dst, Address(dst, OM_OFFSET_NO_MONITOR_VALUE_TAG(header)));
4770 
4771   bind(fast);
4772   shrq(dst, markWord::klass_shift);
4773 }
4774 #endif
4775 
4776 void MacroAssembler::load_klass(Register dst, Register src, Register tmp, bool null_check_src) {
4777   assert_different_registers(src, tmp);
4778   assert_different_registers(dst, tmp);
4779   if (null_check_src) {
4780     if (UseCompactObjectHeaders) {
4781       null_check(src, oopDesc::mark_offset_in_bytes());
4782     } else {
4783       null_check(src, oopDesc::klass_offset_in_bytes());
4784     }
4785   }
4786 #ifdef _LP64
4787   if (UseCompressedClassPointers) {
4788     load_nklass(dst, src);
4789     decode_klass_not_null(dst, tmp);
4790   } else
4791 #endif
4792     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4793 }
4794 
4795 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
4796   load_klass(dst, src, tmp);
4797   movptr(dst, Address(dst, Klass::prototype_header_offset()));
4798 }
4799 
4800 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
4801   assert(!UseCompactObjectHeaders, "not with compact headers");
4802   assert_different_registers(src, tmp);
4803   assert_different_registers(dst, tmp);
4804 #ifdef _LP64
4805   if (UseCompressedClassPointers) {
4806     encode_klass_not_null(src, tmp);
4807     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4808   } else
4809 #endif
4810    movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4811 }
4812 
4813 void MacroAssembler::cmp_klass(Register klass, Register obj, Register tmp) {
4814 #ifdef _LP64
4815   if (UseCompactObjectHeaders) {
4816     // NOTE: We need to deal with possible ObjectMonitor in object header.
4817     // Eventually we might be able to do simple movl & cmpl like in
4818     // the CCP path below.
4819     load_nklass(tmp, obj);
4820     cmpl(klass, tmp);
4821   } else if (UseCompressedClassPointers) {
4822     cmpl(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
4823   } else
4824 #endif
4825   {
4826     cmpptr(klass, Address(obj, oopDesc::klass_offset_in_bytes()));
4827   }
4828 }
4829 
4830 void MacroAssembler::cmp_klass(Register src, Register dst, Register tmp1, Register tmp2) {
4831 #ifdef _LP64
4832   if (UseCompactObjectHeaders) {
4833     // NOTE: We need to deal with possible ObjectMonitor in object header.
4834     // Eventually we might be able to do simple movl & cmpl like in
4835     // the CCP path below.
4836     assert(tmp2 != noreg, "need tmp2");
4837     assert_different_registers(src, dst, tmp1, tmp2);
4838     load_nklass(tmp1, src);
4839     load_nklass(tmp2, dst);
4840     cmpl(tmp1, tmp2);
4841   } else if (UseCompressedClassPointers) {
4842     movl(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
4843     cmpl(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
4844   } else
4845 #endif
4846   {
4847     movptr(tmp1, Address(src, oopDesc::klass_offset_in_bytes()));
4848     cmpptr(tmp1, Address(dst, oopDesc::klass_offset_in_bytes()));
4849   }
4850 }
4851 
4852 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
4853                                     Register tmp1, Register thread_tmp) {
4854   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4855   decorators = AccessInternal::decorator_fixup(decorators);
4856   bool as_raw = (decorators & AS_RAW) != 0;
4857   if (as_raw) {
4858     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4859   } else {
4860     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4861   }
4862 }
4863 
4864 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
4865                                      Register tmp1, Register tmp2) {
4866   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4867   decorators = AccessInternal::decorator_fixup(decorators);
4868   bool as_raw = (decorators & AS_RAW) != 0;
4869   if (as_raw) {

5189   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5190 }
5191 
5192 void MacroAssembler::reinit_heapbase() {
5193   if (UseCompressedOops) {
5194     if (Universe::heap() != NULL) {
5195       if (CompressedOops::base() == NULL) {
5196         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
5197       } else {
5198         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
5199       }
5200     } else {
5201       movptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
5202     }
5203   }
5204 }
5205 
5206 #endif // _LP64
5207 
5208 // C2 compiled method's prolog code.
5209 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub, int max_monitors) {
5210 
5211   // WARNING: Initial instruction MUST be 5 bytes or longer so that
5212   // NativeJump::patch_verified_entry will be able to patch out the entry
5213   // code safely. The push to verify stack depth is ok at 5 bytes,
5214   // the frame allocation can be either 3 or 6 bytes. So if we don't do
5215   // stack bang then we must use the 6 byte frame allocation even if
5216   // we have no frame. :-(
5217   assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
5218 
5219   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
5220   // Remove word for return addr
5221   framesize -= wordSize;
5222   stack_bang_size -= wordSize;
5223 
5224   // Calls to C2R adapters often do not accept exceptional returns.
5225   // We require that their callers must bang for them.  But be careful, because
5226   // some VM calls (such as call site linkage) can use several kilobytes of
5227   // stack.  But the stack safety zone should account for that.
5228   // See bugs 4446381, 4468289, 4497237.
5229   if (stack_bang_size > 0) {

5271   }
5272   if (UseSSE >= 2 && VerifyFPU) {
5273     verify_FPU(0, "FPU stack must be clean on entry");
5274   }
5275 #endif
5276 
5277 #ifdef ASSERT
5278   if (VerifyStackAtCalls) {
5279     Label L;
5280     push(rax);
5281     mov(rax, rsp);
5282     andptr(rax, StackAlignmentInBytes-1);
5283     cmpptr(rax, StackAlignmentInBytes-wordSize);
5284     pop(rax);
5285     jcc(Assembler::equal, L);
5286     STOP("Stack is not properly aligned!");
5287     bind(L);
5288   }
5289 #endif
5290 
5291 #if defined(_LP64) && defined(COMPILER2)
5292   if (UseFastLocking && max_monitors > 0) {
5293     C2CheckLockStackStub* stub = new (Compile::current()->comp_arena()) C2CheckLockStackStub();
5294     Compile::current()->output()->add_stub(stub);
5295     assert(!is_stub, "only methods have monitors");
5296     Register thread = r15_thread;
5297     movptr(rax, Address(thread, JavaThread::lock_stack_current_offset()));
5298     addptr(rax, max_monitors * wordSize);
5299     cmpptr(rax, Address(thread, JavaThread::lock_stack_limit_offset()));
5300     jcc(Assembler::greaterEqual, stub->entry());
5301     bind(stub->continuation());
5302   }
5303 #endif
5304 
5305   if (!is_stub) {
5306     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5307     bs->nmethod_entry_barrier(this);
5308   }
5309 }
5310 
5311 #if COMPILER2_OR_JVMCI
5312 
5313 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
5314 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5315   // cnt - number of qwords (8-byte words).
5316   // base - start address, qword aligned.
5317   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
5318   bool use64byteVector = MaxVectorSize == 64 && AVX3Threshold == 0;
5319   if (use64byteVector) {
5320     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
5321   } else if (MaxVectorSize >= 32) {
5322     vpxor(xtmp, xtmp, xtmp, AVX_256bit);
5323   } else {
5324     pxor(xtmp, xtmp);

8767 
8768   MacroAssembler::call_VM_leaf_base(CAST_FROM_FN_PTR(address, Thread::current), 0);
8769 
8770 #ifdef _LP64
8771   pop(r11);
8772   pop(r10);
8773   pop(r9);
8774   pop(r8);
8775 #endif
8776   pop(rcx);
8777   pop(rdx);
8778   LP64_ONLY(pop(rsi);)
8779   LP64_ONLY(pop(rdi);)
8780   if (thread != rax) {
8781     mov(thread, rax);
8782     pop(rax);
8783   }
8784 }
8785 
8786 #endif // !WIN32 || _LP64
8787 
8788 void MacroAssembler::fast_lock_impl(Register obj, Register hdr, Register thread, Register tmp, Label& slow, bool rt_check_stack) {
8789   assert(hdr == rax, "header must be in rax for cmpxchg");
8790   assert_different_registers(obj, hdr, thread, tmp);
8791 
8792   // First we need to check if the lock-stack has room for pushing the object reference.
8793   if (rt_check_stack) {
8794     movptr(tmp, Address(thread, JavaThread::lock_stack_current_offset()));
8795     cmpptr(tmp, Address(thread, JavaThread::lock_stack_limit_offset()));
8796     jcc(Assembler::greaterEqual, slow);
8797   }
8798 #ifdef ASSERT
8799   else {
8800     Label ok;
8801     movptr(tmp, Address(thread, JavaThread::lock_stack_current_offset()));
8802     cmpptr(tmp, Address(thread, JavaThread::lock_stack_limit_offset()));
8803     jcc(Assembler::less, ok);
8804     stop("Not enough room in lock stack; should have been checked in the method prologue");
8805     bind(ok);
8806   }
8807 #endif
8808 
8809   // Now we attempt to take the fast-lock.
8810   // Clear lowest two header bits (locked state).
8811   andptr(hdr, ~(int32_t )markWord::lock_mask_in_place);
8812   movptr(tmp, hdr);
8813   // Set lowest bit (unlocked state).
8814   orptr(hdr, markWord::unlocked_value);
8815   lock();
8816   cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
8817   jcc(Assembler::notEqual, slow);
8818 
8819   // If successful, push object to lock-stack.
8820   movptr(tmp, Address(thread, JavaThread::lock_stack_current_offset()));
8821   movptr(Address(tmp, 0), obj);
8822   addptr(tmp, oopSize);
8823   movptr(Address(thread, JavaThread::lock_stack_current_offset()), tmp);
8824 }
8825 
8826 void MacroAssembler::fast_unlock_impl(Register obj, Register hdr, Register tmp, Label& slow) {
8827   assert(hdr == rax, "header must be in rax for cmpxchg");
8828   assert_different_registers(obj, hdr, tmp);
8829 
8830   // Mark-word must be 00 now, try to swing it back to 01 (unlocked)
8831   movptr(tmp, hdr); // The expected old value
8832   orptr(tmp, markWord::unlocked_value);
8833   lock();
8834   cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
8835   jcc(Assembler::notEqual, slow);
8836   // Pop the lock object from the lock-stack.
8837 #ifdef _LP64
8838   const Register thread = r15_thread;
8839 #else
8840   const Register thread = rax;
8841   get_thread(rax);
8842 #endif
8843   subptr(Address(thread, JavaThread::lock_stack_current_offset()), oopSize);
8844 }
< prev index next >