< prev index next >

src/hotspot/cpu/x86/macroAssembler_x86.cpp

Print this page

   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "jvm.h"
   27 #include "asm/assembler.hpp"
   28 #include "asm/assembler.inline.hpp"
   29 #include "c1/c1_FrameMap.hpp"
   30 #include "compiler/compiler_globals.hpp"
   31 #include "compiler/disassembler.hpp"

   32 #include "gc/shared/barrierSet.hpp"
   33 #include "gc/shared/barrierSetAssembler.hpp"
   34 #include "gc/shared/collectedHeap.inline.hpp"
   35 #include "gc/shared/tlab_globals.hpp"
   36 #include "interpreter/bytecodeHistogram.hpp"
   37 #include "interpreter/interpreter.hpp"
   38 #include "memory/resourceArea.hpp"
   39 #include "memory/universe.hpp"
   40 #include "oops/accessDecorators.hpp"
   41 #include "oops/compressedOops.inline.hpp"
   42 #include "oops/klass.inline.hpp"
   43 #include "prims/methodHandles.hpp"
   44 #include "runtime/flags/flagSetting.hpp"
   45 #include "runtime/interfaceSupport.inline.hpp"
   46 #include "runtime/jniHandles.hpp"
   47 #include "runtime/objectMonitor.hpp"
   48 #include "runtime/os.hpp"
   49 #include "runtime/safepoint.hpp"
   50 #include "runtime/safepointMechanism.hpp"
   51 #include "runtime/sharedRuntime.hpp"

   52 #include "runtime/stubRoutines.hpp"
   53 #include "runtime/thread.hpp"
   54 #include "utilities/macros.hpp"

   55 #include "crc32c.h"



   56 
   57 #ifdef PRODUCT
   58 #define BLOCK_COMMENT(str) /* nothing */
   59 #define STOP(error) stop(error)
   60 #else
   61 #define BLOCK_COMMENT(str) block_comment(str)
   62 #define STOP(error) block_comment(error); stop(error)
   63 #endif
   64 
   65 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   66 
   67 #ifdef ASSERT
   68 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   69 #endif
   70 
   71 static Assembler::Condition reverse[] = {
   72     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   73     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   74     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   75     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,

 1619 }
 1620 
 1621 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1622 
 1623   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
 1624   pass_arg1(this, arg_1);
 1625   pass_arg0(this, arg_0);
 1626   call_VM_leaf(entry_point, 2);
 1627 }
 1628 
 1629 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1630   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
 1631   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
 1632   pass_arg2(this, arg_2);
 1633   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
 1634   pass_arg1(this, arg_1);
 1635   pass_arg0(this, arg_0);
 1636   call_VM_leaf(entry_point, 3);
 1637 }
 1638 




 1639 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1640   pass_arg0(this, arg_0);
 1641   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1642 }
 1643 
 1644 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1645 
 1646   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
 1647   pass_arg1(this, arg_1);
 1648   pass_arg0(this, arg_0);
 1649   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1650 }
 1651 
 1652 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1653   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
 1654   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
 1655   pass_arg2(this, arg_2);
 1656   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
 1657   pass_arg1(this, arg_1);
 1658   pass_arg0(this, arg_0);

 2713     lea(rscratch1, src);
 2714     Assembler::mulss(dst, Address(rscratch1, 0));
 2715   }
 2716 }
 2717 
 2718 void MacroAssembler::null_check(Register reg, int offset) {
 2719   if (needs_explicit_null_check(offset)) {
 2720     // provoke OS NULL exception if reg = NULL by
 2721     // accessing M[reg] w/o changing any (non-CC) registers
 2722     // NOTE: cmpl is plenty here to provoke a segv
 2723     cmpptr(rax, Address(reg, 0));
 2724     // Note: should probably use testl(rax, Address(reg, 0));
 2725     //       may be shorter code (however, this version of
 2726     //       testl needs to be implemented first)
 2727   } else {
 2728     // nothing to do, (later) access of M[reg + offset]
 2729     // will provoke OS NULL exception if reg = NULL
 2730   }
 2731 }
 2732 












































































































































 2733 void MacroAssembler::os_breakpoint() {
 2734   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 2735   // (e.g., MSVC can't call ps() otherwise)
 2736   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 2737 }
 2738 
 2739 void MacroAssembler::unimplemented(const char* what) {
 2740   const char* buf = NULL;
 2741   {
 2742     ResourceMark rm;
 2743     stringStream ss;
 2744     ss.print("unimplemented: %s", what);
 2745     buf = code_string(ss.as_string());
 2746   }
 2747   stop(buf);
 2748 }
 2749 
 2750 #ifdef _LP64
 2751 #define XSTATE_BV 0x200
 2752 #endif

 3551 }
 3552 
 3553 // C++ bool manipulation
 3554 void MacroAssembler::testbool(Register dst) {
 3555   if(sizeof(bool) == 1)
 3556     testb(dst, 0xff);
 3557   else if(sizeof(bool) == 2) {
 3558     // testw implementation needed for two byte bools
 3559     ShouldNotReachHere();
 3560   } else if(sizeof(bool) == 4)
 3561     testl(dst, dst);
 3562   else
 3563     // unsupported
 3564     ShouldNotReachHere();
 3565 }
 3566 
 3567 void MacroAssembler::testptr(Register dst, Register src) {
 3568   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 3569 }
 3570 























































































































 3571 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 3572 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 3573                                    Register var_size_in_bytes,
 3574                                    int con_size_in_bytes,
 3575                                    Register t1,
 3576                                    Register t2,
 3577                                    Label& slow_case) {
 3578   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 3579   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 3580 }
 3581 
 3582 RegSet MacroAssembler::call_clobbered_gp_registers() {
 3583   RegSet regs;
 3584 #ifdef _LP64
 3585   regs += RegSet::of(rax, rcx, rdx);
 3586 #ifndef WINDOWS
 3587   regs += RegSet::of(rsi, rdi);
 3588 #endif
 3589   regs += RegSet::range(r8, r11);
 3590 #else

 3812     // clear topmost word (no jump would be needed if conditional assignment worked here)
 3813     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 3814     // index could be 0 now, must check again
 3815     jcc(Assembler::zero, done);
 3816     bind(even);
 3817   }
 3818 #endif // !_LP64
 3819   // initialize remaining object fields: index is a multiple of 2 now
 3820   {
 3821     Label loop;
 3822     bind(loop);
 3823     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 3824     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 3825     decrement(index);
 3826     jcc(Assembler::notZero, loop);
 3827   }
 3828 
 3829   bind(done);
 3830 }
 3831 


















































 3832 // Look up the method for a megamorphic invokeinterface call.
 3833 // The target method is determined by <intf_klass, itable_index>.
 3834 // The receiver klass is in recv_klass.
 3835 // On success, the result will be in method_result, and execution falls through.
 3836 // On failure, execution transfers to the given label.
 3837 void MacroAssembler::lookup_interface_method(Register recv_klass,
 3838                                              Register intf_klass,
 3839                                              RegisterOrConstant itable_index,
 3840                                              Register method_result,
 3841                                              Register scan_temp,
 3842                                              Label& L_no_such_interface,
 3843                                              bool return_method) {
 3844   assert_different_registers(recv_klass, intf_klass, scan_temp);
 3845   assert_different_registers(method_result, intf_klass, scan_temp);
 3846   assert(recv_klass != method_result || !return_method,
 3847          "recv_klass can be destroyed when method isn't needed");
 3848 
 3849   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 3850          "caller must use same register for non-constant itable index as for method");
 3851 

 4160   } else {
 4161     Label L;
 4162     jccb(negate_condition(cc), L);
 4163     movl(dst, src);
 4164     bind(L);
 4165   }
 4166 }
 4167 
 4168 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 4169   if (VM_Version::supports_cmov()) {
 4170     cmovl(cc, dst, src);
 4171   } else {
 4172     Label L;
 4173     jccb(negate_condition(cc), L);
 4174     movl(dst, src);
 4175     bind(L);
 4176   }
 4177 }
 4178 
 4179 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 4180   if (!VerifyOops) return;




 4181 
 4182   // Pass register number to verify_oop_subroutine
 4183   const char* b = NULL;
 4184   {
 4185     ResourceMark rm;
 4186     stringStream ss;
 4187     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 4188     b = code_string(ss.as_string());
 4189   }
 4190   BLOCK_COMMENT("verify_oop {");
 4191 #ifdef _LP64
 4192   push(rscratch1);                    // save r10, trashed by movptr()
 4193 #endif
 4194   push(rax);                          // save rax,
 4195   push(reg);                          // pass register argument
 4196   ExternalAddress buffer((address) b);
 4197   // avoid using pushptr, as it modifies scratch registers
 4198   // and our contract is not to modify anything
 4199   movptr(rax, buffer.addr());
 4200   push(rax);

 4219   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 4220   int stackElementSize = Interpreter::stackElementSize;
 4221   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 4222 #ifdef ASSERT
 4223   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 4224   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 4225 #endif
 4226   Register             scale_reg    = noreg;
 4227   Address::ScaleFactor scale_factor = Address::no_scale;
 4228   if (arg_slot.is_constant()) {
 4229     offset += arg_slot.as_constant() * stackElementSize;
 4230   } else {
 4231     scale_reg    = arg_slot.as_register();
 4232     scale_factor = Address::times(stackElementSize);
 4233   }
 4234   offset += wordSize;           // return PC is on stack
 4235   return Address(rsp, scale_reg, scale_factor, offset);
 4236 }
 4237 
 4238 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 4239   if (!VerifyOops) return;




 4240 
 4241   // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
 4242   // Pass register number to verify_oop_subroutine
 4243   const char* b = NULL;
 4244   {
 4245     ResourceMark rm;
 4246     stringStream ss;
 4247     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 4248     b = code_string(ss.as_string());
 4249   }
 4250 #ifdef _LP64
 4251   push(rscratch1);                    // save r10, trashed by movptr()
 4252 #endif
 4253   push(rax);                          // save rax,
 4254   // addr may contain rsp so we will have to adjust it based on the push
 4255   // we just did (and on 64 bit we do two pushes)
 4256   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 4257   // stores rax into addr which is backwards of what was intended.
 4258   if (addr.uses(rsp)) {
 4259     lea(rax, addr);

 4721 
 4722 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 4723   // get mirror
 4724   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 4725   load_method_holder(mirror, method);
 4726   movptr(mirror, Address(mirror, mirror_offset));
 4727   resolve_oop_handle(mirror, tmp);
 4728 }
 4729 
 4730 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 4731   load_method_holder(rresult, rmethod);
 4732   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 4733 }
 4734 
 4735 void MacroAssembler::load_method_holder(Register holder, Register method) {
 4736   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 4737   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 4738   movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
 4739 }
 4740 








 4741 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 4742   assert_different_registers(src, tmp);
 4743   assert_different_registers(dst, tmp);
 4744 #ifdef _LP64
 4745   if (UseCompressedClassPointers) {
 4746     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 4747     decode_klass_not_null(dst, tmp);
 4748   } else
 4749 #endif
 4750     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));





 4751 }
 4752 
 4753 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 4754   assert_different_registers(src, tmp);
 4755   assert_different_registers(dst, tmp);
 4756 #ifdef _LP64
 4757   if (UseCompressedClassPointers) {
 4758     encode_klass_not_null(src, tmp);
 4759     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 4760   } else
 4761 #endif
 4762     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 4763 }
 4764 
 4765 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 4766                                     Register tmp1, Register thread_tmp) {
 4767   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4768   decorators = AccessInternal::decorator_fixup(decorators);
 4769   bool as_raw = (decorators & AS_RAW) != 0;
 4770   if (as_raw) {
 4771     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 4772   } else {
 4773     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 4774   }
 4775 }
 4776 
 4777 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
 4778                                      Register tmp1, Register tmp2, Register tmp3) {
 4779   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4780   decorators = AccessInternal::decorator_fixup(decorators);
 4781   bool as_raw = (decorators & AS_RAW) != 0;
 4782   if (as_raw) {
 4783     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
 4784   } else {
 4785     bs->store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
 4786   }
 4787 }
 4788 








































 4789 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 4790                                    Register thread_tmp, DecoratorSet decorators) {
 4791   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 4792 }
 4793 
 4794 // Doesn't do verfication, generates fixed size code
 4795 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 4796                                             Register thread_tmp, DecoratorSet decorators) {
 4797   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 4798 }
 4799 
 4800 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
 4801                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 4802   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2, tmp3);
 4803 }
 4804 
 4805 // Used for storing NULLs.
 4806 void MacroAssembler::store_heap_oop_null(Address dst) {
 4807   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 4808 }

 5106   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 5107   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 5108 }
 5109 
 5110 void MacroAssembler::reinit_heapbase() {
 5111   if (UseCompressedOops) {
 5112     if (Universe::heap() != NULL) {
 5113       if (CompressedOops::base() == NULL) {
 5114         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 5115       } else {
 5116         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
 5117       }
 5118     } else {
 5119       movptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
 5120     }
 5121   }
 5122 }
 5123 
 5124 #endif // _LP64
 5125 

 5126 // C2 compiled method's prolog code.
 5127 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub) {




 5128 
 5129   // WARNING: Initial instruction MUST be 5 bytes or longer so that
 5130   // NativeJump::patch_verified_entry will be able to patch out the entry
 5131   // code safely. The push to verify stack depth is ok at 5 bytes,
 5132   // the frame allocation can be either 3 or 6 bytes. So if we don't do
 5133   // stack bang then we must use the 6 byte frame allocation even if
 5134   // we have no frame. :-(
 5135   assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
 5136 
 5137   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 5138   // Remove word for return addr
 5139   framesize -= wordSize;
 5140   stack_bang_size -= wordSize;
 5141 
 5142   // Calls to C2R adapters often do not accept exceptional returns.
 5143   // We require that their callers must bang for them.  But be careful, because
 5144   // some VM calls (such as call site linkage) can use several kilobytes of
 5145   // stack.  But the stack safety zone should account for that.
 5146   // See bugs 4446381, 4468289, 4497237.
 5147   if (stack_bang_size > 0) {

 5160     // Create frame
 5161     if (framesize) {
 5162       subptr(rsp, framesize);
 5163     }
 5164   } else {
 5165     // Create frame (force generation of a 4 byte immediate value)
 5166     subptr_imm32(rsp, framesize);
 5167 
 5168     // Save RBP register now.
 5169     framesize -= wordSize;
 5170     movptr(Address(rsp, framesize), rbp);
 5171     // Save caller's stack pointer into RBP if the frame pointer is preserved.
 5172     if (PreserveFramePointer) {
 5173       movptr(rbp, rsp);
 5174       if (framesize > 0) {
 5175         addptr(rbp, framesize);
 5176       }
 5177     }
 5178   }
 5179 






 5180   if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
 5181     framesize -= wordSize;
 5182     movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
 5183   }
 5184 
 5185 #ifndef _LP64
 5186   // If method sets FPU control word do it now
 5187   if (fp_mode_24b) {
 5188     fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
 5189   }
 5190   if (UseSSE >= 2 && VerifyFPU) {
 5191     verify_FPU(0, "FPU stack must be clean on entry");
 5192   }
 5193 #endif
 5194 
 5195 #ifdef ASSERT
 5196   if (VerifyStackAtCalls) {
 5197     Label L;
 5198     push(rax);
 5199     mov(rax, rsp);
 5200     andptr(rax, StackAlignmentInBytes-1);
 5201     cmpptr(rax, StackAlignmentInBytes-wordSize);
 5202     pop(rax);
 5203     jcc(Assembler::equal, L);
 5204     STOP("Stack is not properly aligned!");
 5205     bind(L);
 5206   }
 5207 #endif
 5208 
 5209   if (!is_stub) {
 5210     BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5211     bs->nmethod_entry_barrier(this);
 5212   }
 5213 }

 5214 
 5215 #if COMPILER2_OR_JVMCI
 5216 
 5217 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 5218 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 5219   // cnt - number of qwords (8-byte words).
 5220   // base - start address, qword aligned.
 5221   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 5222   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 5223   if (use64byteVector) {
 5224     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
 5225   } else if (MaxVectorSize >= 32) {
 5226     vpxor(xtmp, xtmp, xtmp, AVX_256bit);


 5227   } else {
 5228     pxor(xtmp, xtmp);

 5229   }
 5230   jmp(L_zero_64_bytes);
 5231 
 5232   BIND(L_loop);
 5233   if (MaxVectorSize >= 32) {
 5234     fill64(base, 0, xtmp, use64byteVector);
 5235   } else {
 5236     movdqu(Address(base,  0), xtmp);
 5237     movdqu(Address(base, 16), xtmp);
 5238     movdqu(Address(base, 32), xtmp);
 5239     movdqu(Address(base, 48), xtmp);
 5240   }
 5241   addptr(base, 64);
 5242 
 5243   BIND(L_zero_64_bytes);
 5244   subptr(cnt, 8);
 5245   jccb(Assembler::greaterEqual, L_loop);
 5246 
 5247   // Copy trailing 64 bytes
 5248   if (use64byteVector) {
 5249     addptr(cnt, 8);
 5250     jccb(Assembler::equal, L_end);
 5251     fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true);
 5252     jmp(L_end);
 5253   } else {
 5254     addptr(cnt, 4);
 5255     jccb(Assembler::less, L_tail);
 5256     if (MaxVectorSize >= 32) {
 5257       vmovdqu(Address(base, 0), xtmp);
 5258     } else {
 5259       movdqu(Address(base,  0), xtmp);
 5260       movdqu(Address(base, 16), xtmp);
 5261     }
 5262   }
 5263   addptr(base, 32);
 5264   subptr(cnt, 4);
 5265 
 5266   BIND(L_tail);
 5267   addptr(cnt, 4);
 5268   jccb(Assembler::lessEqual, L_end);
 5269   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 5270     fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp);
 5271   } else {
 5272     decrement(cnt);
 5273 
 5274     BIND(L_sloop);
 5275     movq(Address(base, 0), xtmp);
 5276     addptr(base, 8);
 5277     decrement(cnt);
 5278     jccb(Assembler::greaterEqual, L_sloop);
 5279   }
 5280   BIND(L_end);
 5281 }
 5282 















































































































































































































































































































































































































 5283 // Clearing constant sized memory using YMM/ZMM registers.
 5284 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 5285   assert(UseAVX > 2 && VM_Version::supports_avx512vlbw(), "");
 5286   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 5287 
 5288   int vector64_count = (cnt & (~0x7)) >> 3;
 5289   cnt = cnt & 0x7;
 5290 
 5291   // 64 byte initialization loop.
 5292   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 5293   for (int i = 0; i < vector64_count; i++) {
 5294     fill64(base, i * 64, xtmp, use64byteVector);
 5295   }
 5296 
 5297   // Clear remaining 64 byte tail.
 5298   int disp = vector64_count * 64;
 5299   if (cnt) {
 5300     switch (cnt) {
 5301       case 1:
 5302         movq(Address(base, disp), xtmp);

 5334         break;
 5335       case 7:
 5336         if (use64byteVector) {
 5337           movl(rtmp, 0x7F);
 5338           kmovwl(mask, rtmp);
 5339           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, Assembler::AVX_512bit);
 5340         } else {
 5341           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, Assembler::AVX_256bit);
 5342           movl(rtmp, 0x7);
 5343           kmovwl(mask, rtmp);
 5344           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, Assembler::AVX_256bit);
 5345         }
 5346         break;
 5347       default:
 5348         fatal("Unexpected length : %d\n",cnt);
 5349         break;
 5350     }
 5351   }
 5352 }
 5353 
 5354 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp,
 5355                                bool is_large, KRegister mask) {
 5356   // cnt      - number of qwords (8-byte words).
 5357   // base     - start address, qword aligned.
 5358   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 5359   assert(base==rdi, "base register must be edi for rep stos");
 5360   assert(tmp==rax,   "tmp register must be eax for rep stos");
 5361   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 5362   assert(InitArrayShortSize % BytesPerLong == 0,
 5363     "InitArrayShortSize should be the multiple of BytesPerLong");
 5364 
 5365   Label DONE;
 5366   if (!is_large || !UseXMMForObjInit) {
 5367     xorptr(tmp, tmp);
 5368   }
 5369 
 5370   if (!is_large) {
 5371     Label LOOP, LONG;
 5372     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 5373     jccb(Assembler::greater, LONG);
 5374 
 5375     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 5376 
 5377     decrement(cnt);
 5378     jccb(Assembler::negative, DONE); // Zero length
 5379 
 5380     // Use individual pointer-sized stores for small counts:
 5381     BIND(LOOP);
 5382     movptr(Address(base, cnt, Address::times_ptr), tmp);
 5383     decrement(cnt);
 5384     jccb(Assembler::greaterEqual, LOOP);
 5385     jmpb(DONE);
 5386 
 5387     BIND(LONG);
 5388   }
 5389 
 5390   // Use longer rep-prefixed ops for non-small counts:
 5391   if (UseFastStosb) {
 5392     shlptr(cnt, 3); // convert to number of bytes
 5393     rep_stosb();
 5394   } else if (UseXMMForObjInit) {
 5395     xmm_clear_mem(base, cnt, tmp, xtmp, mask);
 5396   } else {
 5397     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 5398     rep_stos();
 5399   }
 5400 
 5401   BIND(DONE);
 5402 }
 5403 
 5404 #endif //COMPILER2_OR_JVMCI
 5405 
 5406 
 5407 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 5408                                    Register to, Register value, Register count,
 5409                                    Register rtmp, XMMRegister xtmp) {
 5410   ShortBranchVerifier sbv(this);
 5411   assert_different_registers(to, value, count, rtmp);
 5412   Label L_exit;
 5413   Label L_fill_2_bytes, L_fill_4_bytes;
 5414 
 5415 #if defined(COMPILER2) && defined(_LP64)

   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "jvm.h"
   27 #include "asm/assembler.hpp"
   28 #include "asm/assembler.inline.hpp"
   29 #include "c1/c1_FrameMap.hpp"
   30 #include "compiler/compiler_globals.hpp"
   31 #include "compiler/disassembler.hpp"
   32 #include "ci/ciInlineKlass.hpp"
   33 #include "gc/shared/barrierSet.hpp"
   34 #include "gc/shared/barrierSetAssembler.hpp"
   35 #include "gc/shared/collectedHeap.inline.hpp"
   36 #include "gc/shared/tlab_globals.hpp"
   37 #include "interpreter/bytecodeHistogram.hpp"
   38 #include "interpreter/interpreter.hpp"
   39 #include "memory/resourceArea.hpp"
   40 #include "memory/universe.hpp"
   41 #include "oops/accessDecorators.hpp"
   42 #include "oops/compressedOops.inline.hpp"
   43 #include "oops/klass.inline.hpp"
   44 #include "prims/methodHandles.hpp"
   45 #include "runtime/flags/flagSetting.hpp"
   46 #include "runtime/interfaceSupport.inline.hpp"
   47 #include "runtime/jniHandles.hpp"
   48 #include "runtime/objectMonitor.hpp"
   49 #include "runtime/os.hpp"
   50 #include "runtime/safepoint.hpp"
   51 #include "runtime/safepointMechanism.hpp"
   52 #include "runtime/sharedRuntime.hpp"
   53 #include "runtime/signature_cc.hpp"
   54 #include "runtime/stubRoutines.hpp"
   55 #include "runtime/thread.hpp"
   56 #include "utilities/macros.hpp"
   57 #include "vmreg_x86.inline.hpp"
   58 #include "crc32c.h"
   59 #ifdef COMPILER2
   60 #include "opto/output.hpp"
   61 #endif
   62 
   63 #ifdef PRODUCT
   64 #define BLOCK_COMMENT(str) /* nothing */
   65 #define STOP(error) stop(error)
   66 #else
   67 #define BLOCK_COMMENT(str) block_comment(str)
   68 #define STOP(error) block_comment(error); stop(error)
   69 #endif
   70 
   71 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   72 
   73 #ifdef ASSERT
   74 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   75 #endif
   76 
   77 static Assembler::Condition reverse[] = {
   78     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   79     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   80     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   81     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,

 1625 }
 1626 
 1627 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1628 
 1629   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
 1630   pass_arg1(this, arg_1);
 1631   pass_arg0(this, arg_0);
 1632   call_VM_leaf(entry_point, 2);
 1633 }
 1634 
 1635 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1636   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
 1637   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
 1638   pass_arg2(this, arg_2);
 1639   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
 1640   pass_arg1(this, arg_1);
 1641   pass_arg0(this, arg_0);
 1642   call_VM_leaf(entry_point, 3);
 1643 }
 1644 
 1645 void MacroAssembler::super_call_VM_leaf(address entry_point) {
 1646   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1647 }
 1648 
 1649 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1650   pass_arg0(this, arg_0);
 1651   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1652 }
 1653 
 1654 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1655 
 1656   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
 1657   pass_arg1(this, arg_1);
 1658   pass_arg0(this, arg_0);
 1659   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1660 }
 1661 
 1662 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1663   LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
 1664   LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
 1665   pass_arg2(this, arg_2);
 1666   LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
 1667   pass_arg1(this, arg_1);
 1668   pass_arg0(this, arg_0);

 2723     lea(rscratch1, src);
 2724     Assembler::mulss(dst, Address(rscratch1, 0));
 2725   }
 2726 }
 2727 
 2728 void MacroAssembler::null_check(Register reg, int offset) {
 2729   if (needs_explicit_null_check(offset)) {
 2730     // provoke OS NULL exception if reg = NULL by
 2731     // accessing M[reg] w/o changing any (non-CC) registers
 2732     // NOTE: cmpl is plenty here to provoke a segv
 2733     cmpptr(rax, Address(reg, 0));
 2734     // Note: should probably use testl(rax, Address(reg, 0));
 2735     //       may be shorter code (however, this version of
 2736     //       testl needs to be implemented first)
 2737   } else {
 2738     // nothing to do, (later) access of M[reg + offset]
 2739     // will provoke OS NULL exception if reg = NULL
 2740   }
 2741 }
 2742 
 2743 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
 2744   andptr(markword, markWord::inline_type_mask_in_place);
 2745   cmpptr(markword, markWord::inline_type_pattern);
 2746   jcc(Assembler::equal, is_inline_type);
 2747 }
 2748 
 2749 void MacroAssembler::test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type) {
 2750   movl(temp_reg, Address(klass, Klass::access_flags_offset()));
 2751   testl(temp_reg, JVM_ACC_VALUE);
 2752   jcc(Assembler::notZero, is_inline_type);
 2753 }
 2754 
 2755 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type) {
 2756   testptr(object, object);
 2757   jcc(Assembler::zero, not_inline_type);
 2758   const int is_inline_type_mask = markWord::inline_type_pattern;
 2759   movptr(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
 2760   andptr(tmp, is_inline_type_mask);
 2761   cmpptr(tmp, is_inline_type_mask);
 2762   jcc(Assembler::notEqual, not_inline_type);
 2763 }
 2764 
 2765 void MacroAssembler::test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type) {
 2766 #ifdef ASSERT
 2767   {
 2768     Label done_check;
 2769     test_klass_is_inline_type(klass, temp_reg, done_check);
 2770     stop("test_klass_is_empty_inline_type with non inline type klass");
 2771     bind(done_check);
 2772   }
 2773 #endif
 2774   movl(temp_reg, Address(klass, InstanceKlass::misc_flags_offset()));
 2775   testl(temp_reg, InstanceKlass::misc_flag_is_empty_inline_type());
 2776   jcc(Assembler::notZero, is_empty_inline_type);
 2777 }
 2778 
 2779 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
 2780   movl(temp_reg, flags);
 2781   shrl(temp_reg, ConstantPoolCacheEntry::is_null_free_inline_type_shift);
 2782   andl(temp_reg, 0x1);
 2783   testl(temp_reg, temp_reg);
 2784   jcc(Assembler::notZero, is_null_free_inline_type);
 2785 }
 2786 
 2787 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
 2788   movl(temp_reg, flags);
 2789   shrl(temp_reg, ConstantPoolCacheEntry::is_null_free_inline_type_shift);
 2790   andl(temp_reg, 0x1);
 2791   testl(temp_reg, temp_reg);
 2792   jcc(Assembler::zero, not_null_free_inline_type);
 2793 }
 2794 
 2795 void MacroAssembler::test_field_is_inlined(Register flags, Register temp_reg, Label& is_inlined) {
 2796   movl(temp_reg, flags);
 2797   shrl(temp_reg, ConstantPoolCacheEntry::is_inlined_shift);
 2798   andl(temp_reg, 0x1);
 2799   testl(temp_reg, temp_reg);
 2800   jcc(Assembler::notZero, is_inlined);
 2801 }
 2802 
 2803 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label) {
 2804   Label test_mark_word;
 2805   // load mark word
 2806   movptr(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
 2807   // check displaced
 2808   testl(temp_reg, markWord::unlocked_value);
 2809   jccb(Assembler::notZero, test_mark_word);
 2810   // slow path use klass prototype
 2811   push(rscratch1);
 2812   load_prototype_header(temp_reg, oop, rscratch1);
 2813   pop(rscratch1);
 2814 
 2815   bind(test_mark_word);
 2816   testl(temp_reg, test_bit);
 2817   jcc((jmp_set) ? Assembler::notZero : Assembler::zero, jmp_label);
 2818 }
 2819 
 2820 void MacroAssembler::test_flattened_array_oop(Register oop, Register temp_reg,
 2821                                               Label&is_flattened_array) {
 2822 #ifdef _LP64
 2823   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flattened_array);
 2824 #else
 2825   load_klass(temp_reg, oop, noreg);
 2826   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 2827   test_flattened_array_layout(temp_reg, is_flattened_array);
 2828 #endif
 2829 }
 2830 
 2831 void MacroAssembler::test_non_flattened_array_oop(Register oop, Register temp_reg,
 2832                                                   Label&is_non_flattened_array) {
 2833 #ifdef _LP64
 2834   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flattened_array);
 2835 #else
 2836   load_klass(temp_reg, oop, noreg);
 2837   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 2838   test_non_flattened_array_layout(temp_reg, is_non_flattened_array);
 2839 #endif
 2840 }
 2841 
 2842 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label&is_null_free_array) {
 2843 #ifdef _LP64
 2844   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
 2845 #else
 2846   load_klass(temp_reg, oop, noreg);
 2847   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 2848   test_null_free_array_layout(temp_reg, is_null_free_array);
 2849 #endif
 2850 }
 2851 
 2852 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
 2853 #ifdef _LP64
 2854   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
 2855 #else
 2856   load_klass(temp_reg, oop, noreg);
 2857   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 2858   test_non_null_free_array_layout(temp_reg, is_non_null_free_array);
 2859 #endif
 2860 }
 2861 
 2862 void MacroAssembler::test_flattened_array_layout(Register lh, Label& is_flattened_array) {
 2863   testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
 2864   jcc(Assembler::notZero, is_flattened_array);
 2865 }
 2866 
 2867 void MacroAssembler::test_non_flattened_array_layout(Register lh, Label& is_non_flattened_array) {
 2868   testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
 2869   jcc(Assembler::zero, is_non_flattened_array);
 2870 }
 2871 
 2872 void MacroAssembler::test_null_free_array_layout(Register lh, Label& is_null_free_array) {
 2873   testl(lh, Klass::_lh_null_free_array_bit_inplace);
 2874   jcc(Assembler::notZero, is_null_free_array);
 2875 }
 2876 
 2877 void MacroAssembler::test_non_null_free_array_layout(Register lh, Label& is_non_null_free_array) {
 2878   testl(lh, Klass::_lh_null_free_array_bit_inplace);
 2879   jcc(Assembler::zero, is_non_null_free_array);
 2880 }
 2881 
 2882 
 2883 void MacroAssembler::os_breakpoint() {
 2884   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 2885   // (e.g., MSVC can't call ps() otherwise)
 2886   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 2887 }
 2888 
 2889 void MacroAssembler::unimplemented(const char* what) {
 2890   const char* buf = NULL;
 2891   {
 2892     ResourceMark rm;
 2893     stringStream ss;
 2894     ss.print("unimplemented: %s", what);
 2895     buf = code_string(ss.as_string());
 2896   }
 2897   stop(buf);
 2898 }
 2899 
 2900 #ifdef _LP64
 2901 #define XSTATE_BV 0x200
 2902 #endif

 3701 }
 3702 
 3703 // C++ bool manipulation
 3704 void MacroAssembler::testbool(Register dst) {
 3705   if(sizeof(bool) == 1)
 3706     testb(dst, 0xff);
 3707   else if(sizeof(bool) == 2) {
 3708     // testw implementation needed for two byte bools
 3709     ShouldNotReachHere();
 3710   } else if(sizeof(bool) == 4)
 3711     testl(dst, dst);
 3712   else
 3713     // unsupported
 3714     ShouldNotReachHere();
 3715 }
 3716 
 3717 void MacroAssembler::testptr(Register dst, Register src) {
 3718   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 3719 }
 3720 
 3721 // Object / value buffer allocation...
 3722 //
 3723 // Kills klass and rsi on LP64
 3724 void MacroAssembler::allocate_instance(Register klass, Register new_obj,
 3725                                        Register t1, Register t2,
 3726                                        bool clear_fields, Label& alloc_failed)
 3727 {
 3728   Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop;
 3729   Register layout_size = t1;
 3730   assert(new_obj == rax, "needs to be rax, according to barrier asm eden_allocate");
 3731   assert_different_registers(klass, new_obj, t1, t2);
 3732 
 3733   // get instance_size in InstanceKlass (scaled to a count of bytes)
 3734   movl(layout_size, Address(klass, Klass::layout_helper_offset()));
 3735   // test to see if it has a finalizer or is malformed in some way
 3736   testl(layout_size, Klass::_lh_instance_slow_path_bit);
 3737   jcc(Assembler::notZero, slow_case_no_pop);
 3738 
 3739   // Allocate the instance:
 3740   //  If TLAB is enabled:
 3741   //    Try to allocate in the TLAB.
 3742   //    If fails, go to the slow path.
 3743   //  Else If inline contiguous allocations are enabled:
 3744   //    Try to allocate in eden.
 3745   //    If fails due to heap end, go to slow path.
 3746   //
 3747   //  If TLAB is enabled OR inline contiguous is enabled:
 3748   //    Initialize the allocation.
 3749   //    Exit.
 3750   //
 3751   //  Go to slow path.
 3752   const bool allow_shared_alloc =
 3753     Universe::heap()->supports_inline_contig_alloc();
 3754 
 3755   push(klass);
 3756   const Register thread = LP64_ONLY(r15_thread) NOT_LP64(klass);
 3757 #ifndef _LP64
 3758   if (UseTLAB || allow_shared_alloc) {
 3759     get_thread(thread);
 3760   }
 3761 #endif // _LP64
 3762 
 3763   if (UseTLAB) {
 3764     tlab_allocate(thread, new_obj, layout_size, 0, klass, t2, slow_case);
 3765     if (ZeroTLAB || (!clear_fields)) {
 3766       // the fields have been already cleared
 3767       jmp(initialize_header);
 3768     } else {
 3769       // initialize both the header and fields
 3770       jmp(initialize_object);
 3771     }
 3772   } else {
 3773     // Allocation in the shared Eden, if allowed.
 3774     //
 3775     eden_allocate(thread, new_obj, layout_size, 0, t2, slow_case);
 3776   }
 3777 
 3778   // If UseTLAB or allow_shared_alloc are true, the object is created above and
 3779   // there is an initialize need. Otherwise, skip and go to the slow path.
 3780   if (UseTLAB || allow_shared_alloc) {
 3781     if (clear_fields) {
 3782       // The object is initialized before the header.  If the object size is
 3783       // zero, go directly to the header initialization.
 3784       bind(initialize_object);
 3785       decrement(layout_size, sizeof(oopDesc));
 3786       jcc(Assembler::zero, initialize_header);
 3787 
 3788       // Initialize topmost object field, divide size by 8, check if odd and
 3789       // test if zero.
 3790       Register zero = klass;
 3791       xorl(zero, zero);    // use zero reg to clear memory (shorter code)
 3792       shrl(layout_size, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd
 3793 
 3794   #ifdef ASSERT
 3795       // make sure instance_size was multiple of 8
 3796       Label L;
 3797       // Ignore partial flag stall after shrl() since it is debug VM
 3798       jcc(Assembler::carryClear, L);
 3799       stop("object size is not multiple of 2 - adjust this code");
 3800       bind(L);
 3801       // must be > 0, no extra check needed here
 3802   #endif
 3803 
 3804       // initialize remaining object fields: instance_size was a multiple of 8
 3805       {
 3806         Label loop;
 3807         bind(loop);
 3808         movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 1*oopSize), zero);
 3809         NOT_LP64(movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 2*oopSize), zero));
 3810         decrement(layout_size);
 3811         jcc(Assembler::notZero, loop);
 3812       }
 3813     } // clear_fields
 3814 
 3815     // initialize object header only.
 3816     bind(initialize_header);
 3817     pop(klass);
 3818     Register mark_word = t2;
 3819     movptr(mark_word, Address(klass, Klass::prototype_header_offset()));
 3820     movptr(Address(new_obj, oopDesc::mark_offset_in_bytes ()), mark_word);
 3821 #ifdef _LP64
 3822     xorl(rsi, rsi);                 // use zero reg to clear memory (shorter code)
 3823     store_klass_gap(new_obj, rsi);  // zero klass gap for compressed oops
 3824 #endif
 3825     movptr(t2, klass);         // preserve klass
 3826     Register tmp_store_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
 3827     store_klass(new_obj, t2, tmp_store_klass);  // src klass reg is potentially compressed
 3828 
 3829     jmp(done);
 3830   }
 3831 
 3832   bind(slow_case);
 3833   pop(klass);
 3834   bind(slow_case_no_pop);
 3835   jmp(alloc_failed);
 3836 
 3837   bind(done);
 3838 }
 3839 
 3840 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 3841 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 3842                                    Register var_size_in_bytes,
 3843                                    int con_size_in_bytes,
 3844                                    Register t1,
 3845                                    Register t2,
 3846                                    Label& slow_case) {
 3847   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 3848   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 3849 }
 3850 
 3851 RegSet MacroAssembler::call_clobbered_gp_registers() {
 3852   RegSet regs;
 3853 #ifdef _LP64
 3854   regs += RegSet::of(rax, rcx, rdx);
 3855 #ifndef WINDOWS
 3856   regs += RegSet::of(rsi, rdi);
 3857 #endif
 3858   regs += RegSet::range(r8, r11);
 3859 #else

 4081     // clear topmost word (no jump would be needed if conditional assignment worked here)
 4082     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 4083     // index could be 0 now, must check again
 4084     jcc(Assembler::zero, done);
 4085     bind(even);
 4086   }
 4087 #endif // !_LP64
 4088   // initialize remaining object fields: index is a multiple of 2 now
 4089   {
 4090     Label loop;
 4091     bind(loop);
 4092     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 4093     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 4094     decrement(index);
 4095     jcc(Assembler::notZero, loop);
 4096   }
 4097 
 4098   bind(done);
 4099 }
 4100 
 4101 void MacroAssembler::get_inline_type_field_klass(Register klass, Register index, Register inline_klass) {
 4102   movptr(inline_klass, Address(klass, InstanceKlass::inline_type_field_klasses_offset()));
 4103 #ifdef ASSERT
 4104   {
 4105     Label done;
 4106     cmpptr(inline_klass, 0);
 4107     jcc(Assembler::notEqual, done);
 4108     stop("get_inline_type_field_klass contains no inline klass");
 4109     bind(done);
 4110   }
 4111 #endif
 4112   movptr(inline_klass, Address(inline_klass, index, Address::times_ptr));
 4113 }
 4114 
 4115 void MacroAssembler::get_default_value_oop(Register inline_klass, Register temp_reg, Register obj) {
 4116 #ifdef ASSERT
 4117   {
 4118     Label done_check;
 4119     test_klass_is_inline_type(inline_klass, temp_reg, done_check);
 4120     stop("get_default_value_oop from non inline type klass");
 4121     bind(done_check);
 4122   }
 4123 #endif
 4124   Register offset = temp_reg;
 4125   // Getting the offset of the pre-allocated default value
 4126   movptr(offset, Address(inline_klass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())));
 4127   movl(offset, Address(offset, in_bytes(InlineKlass::default_value_offset_offset())));
 4128 
 4129   // Getting the mirror
 4130   movptr(obj, Address(inline_klass, in_bytes(Klass::java_mirror_offset())));
 4131   resolve_oop_handle(obj, inline_klass);
 4132 
 4133   // Getting the pre-allocated default value from the mirror
 4134   Address field(obj, offset, Address::times_1);
 4135   load_heap_oop(obj, field);
 4136 }
 4137 
 4138 void MacroAssembler::get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj) {
 4139 #ifdef ASSERT
 4140   {
 4141     Label done_check;
 4142     test_klass_is_empty_inline_type(inline_klass, temp_reg, done_check);
 4143     stop("get_empty_value from non-empty inline klass");
 4144     bind(done_check);
 4145   }
 4146 #endif
 4147   get_default_value_oop(inline_klass, temp_reg, obj);
 4148 }
 4149 
 4150 
 4151 // Look up the method for a megamorphic invokeinterface call.
 4152 // The target method is determined by <intf_klass, itable_index>.
 4153 // The receiver klass is in recv_klass.
 4154 // On success, the result will be in method_result, and execution falls through.
 4155 // On failure, execution transfers to the given label.
 4156 void MacroAssembler::lookup_interface_method(Register recv_klass,
 4157                                              Register intf_klass,
 4158                                              RegisterOrConstant itable_index,
 4159                                              Register method_result,
 4160                                              Register scan_temp,
 4161                                              Label& L_no_such_interface,
 4162                                              bool return_method) {
 4163   assert_different_registers(recv_klass, intf_klass, scan_temp);
 4164   assert_different_registers(method_result, intf_klass, scan_temp);
 4165   assert(recv_klass != method_result || !return_method,
 4166          "recv_klass can be destroyed when method isn't needed");
 4167 
 4168   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 4169          "caller must use same register for non-constant itable index as for method");
 4170 

 4479   } else {
 4480     Label L;
 4481     jccb(negate_condition(cc), L);
 4482     movl(dst, src);
 4483     bind(L);
 4484   }
 4485 }
 4486 
 4487 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 4488   if (VM_Version::supports_cmov()) {
 4489     cmovl(cc, dst, src);
 4490   } else {
 4491     Label L;
 4492     jccb(negate_condition(cc), L);
 4493     movl(dst, src);
 4494     bind(L);
 4495   }
 4496 }
 4497 
 4498 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 4499   if (!VerifyOops || VerifyAdapterSharing) {
 4500     // Below address of the code string confuses VerifyAdapterSharing
 4501     // because it may differ between otherwise equivalent adapters.
 4502     return;
 4503   }
 4504 
 4505   // Pass register number to verify_oop_subroutine
 4506   const char* b = NULL;
 4507   {
 4508     ResourceMark rm;
 4509     stringStream ss;
 4510     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 4511     b = code_string(ss.as_string());
 4512   }
 4513   BLOCK_COMMENT("verify_oop {");
 4514 #ifdef _LP64
 4515   push(rscratch1);                    // save r10, trashed by movptr()
 4516 #endif
 4517   push(rax);                          // save rax,
 4518   push(reg);                          // pass register argument
 4519   ExternalAddress buffer((address) b);
 4520   // avoid using pushptr, as it modifies scratch registers
 4521   // and our contract is not to modify anything
 4522   movptr(rax, buffer.addr());
 4523   push(rax);

 4542   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 4543   int stackElementSize = Interpreter::stackElementSize;
 4544   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 4545 #ifdef ASSERT
 4546   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 4547   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 4548 #endif
 4549   Register             scale_reg    = noreg;
 4550   Address::ScaleFactor scale_factor = Address::no_scale;
 4551   if (arg_slot.is_constant()) {
 4552     offset += arg_slot.as_constant() * stackElementSize;
 4553   } else {
 4554     scale_reg    = arg_slot.as_register();
 4555     scale_factor = Address::times(stackElementSize);
 4556   }
 4557   offset += wordSize;           // return PC is on stack
 4558   return Address(rsp, scale_reg, scale_factor, offset);
 4559 }
 4560 
 4561 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 4562   if (!VerifyOops || VerifyAdapterSharing) {
 4563     // Below address of the code string confuses VerifyAdapterSharing
 4564     // because it may differ between otherwise equivalent adapters.
 4565     return;
 4566   }
 4567 
 4568   // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
 4569   // Pass register number to verify_oop_subroutine
 4570   const char* b = NULL;
 4571   {
 4572     ResourceMark rm;
 4573     stringStream ss;
 4574     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 4575     b = code_string(ss.as_string());
 4576   }
 4577 #ifdef _LP64
 4578   push(rscratch1);                    // save r10, trashed by movptr()
 4579 #endif
 4580   push(rax);                          // save rax,
 4581   // addr may contain rsp so we will have to adjust it based on the push
 4582   // we just did (and on 64 bit we do two pushes)
 4583   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 4584   // stores rax into addr which is backwards of what was intended.
 4585   if (addr.uses(rsp)) {
 4586     lea(rax, addr);

 5048 
 5049 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 5050   // get mirror
 5051   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 5052   load_method_holder(mirror, method);
 5053   movptr(mirror, Address(mirror, mirror_offset));
 5054   resolve_oop_handle(mirror, tmp);
 5055 }
 5056 
 5057 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 5058   load_method_holder(rresult, rmethod);
 5059   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 5060 }
 5061 
 5062 void MacroAssembler::load_method_holder(Register holder, Register method) {
 5063   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 5064   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 5065   movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
 5066 }
 5067 
 5068 void MacroAssembler::load_metadata(Register dst, Register src) {
 5069   if (UseCompressedClassPointers) {
 5070     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5071   } else {
 5072     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5073   }
 5074 }
 5075 
 5076 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 5077   assert_different_registers(src, tmp);
 5078   assert_different_registers(dst, tmp);
 5079 #ifdef _LP64
 5080   if (UseCompressedClassPointers) {
 5081     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5082     decode_klass_not_null(dst, tmp);
 5083   } else
 5084 #endif
 5085   movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5086 }
 5087 
 5088 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
 5089   load_klass(dst, src, tmp);
 5090   movptr(dst, Address(dst, Klass::prototype_header_offset()));
 5091 }
 5092 
 5093 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 5094   assert_different_registers(src, tmp);
 5095   assert_different_registers(dst, tmp);
 5096 #ifdef _LP64
 5097   if (UseCompressedClassPointers) {
 5098     encode_klass_not_null(src, tmp);
 5099     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5100   } else
 5101 #endif
 5102     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5103 }
 5104 
 5105 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 5106                                     Register tmp1, Register thread_tmp) {
 5107   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5108   decorators = AccessInternal::decorator_fixup(decorators);
 5109   bool as_raw = (decorators & AS_RAW) != 0;
 5110   if (as_raw) {
 5111     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5112   } else {
 5113     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5114   }
 5115 }
 5116 
 5117 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
 5118                                      Register tmp1, Register tmp2, Register tmp3) {
 5119   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5120   decorators = AccessInternal::decorator_fixup(decorators);
 5121   bool as_raw = (decorators & AS_RAW) != 0;
 5122   if (as_raw) {
 5123     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
 5124   } else {
 5125     bs->store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
 5126   }
 5127 }
 5128 
 5129 void MacroAssembler::access_value_copy(DecoratorSet decorators, Register src, Register dst,
 5130                                        Register inline_klass) {
 5131   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5132   bs->value_copy(this, decorators, src, dst, inline_klass);
 5133 }
 5134 
 5135 void MacroAssembler::first_field_offset(Register inline_klass, Register offset) {
 5136   movptr(offset, Address(inline_klass, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 5137   movl(offset, Address(offset, InlineKlass::first_field_offset_offset()));
 5138 }
 5139 
 5140 void MacroAssembler::data_for_oop(Register oop, Register data, Register inline_klass) {
 5141   // ((address) (void*) o) + vk->first_field_offset();
 5142   Register offset = (data == oop) ? rscratch1 : data;
 5143   first_field_offset(inline_klass, offset);
 5144   if (data == oop) {
 5145     addptr(data, offset);
 5146   } else {
 5147     lea(data, Address(oop, offset));
 5148   }
 5149 }
 5150 
 5151 void MacroAssembler::data_for_value_array_index(Register array, Register array_klass,
 5152                                                 Register index, Register data) {
 5153   assert(index != rcx, "index needs to shift by rcx");
 5154   assert_different_registers(array, array_klass, index);
 5155   assert_different_registers(rcx, array, index);
 5156 
 5157   // array->base() + (index << Klass::layout_helper_log2_element_size(lh));
 5158   movl(rcx, Address(array_klass, Klass::layout_helper_offset()));
 5159 
 5160   // Klass::layout_helper_log2_element_size(lh)
 5161   // (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask;
 5162   shrl(rcx, Klass::_lh_log2_element_size_shift);
 5163   andl(rcx, Klass::_lh_log2_element_size_mask);
 5164   shlptr(index); // index << rcx
 5165 
 5166   lea(data, Address(array, index, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_PRIMITIVE_OBJECT)));
 5167 }
 5168 
 5169 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 5170                                    Register thread_tmp, DecoratorSet decorators) {
 5171   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 5172 }
 5173 
 5174 // Doesn't do verfication, generates fixed size code
 5175 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 5176                                             Register thread_tmp, DecoratorSet decorators) {
 5177   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 5178 }
 5179 
 5180 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
 5181                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 5182   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2, tmp3);
 5183 }
 5184 
 5185 // Used for storing NULLs.
 5186 void MacroAssembler::store_heap_oop_null(Address dst) {
 5187   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 5188 }

 5486   RelocationHolder rspec = metadata_Relocation::spec(klass_index);
 5487   Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
 5488 }
 5489 
 5490 void MacroAssembler::reinit_heapbase() {
 5491   if (UseCompressedOops) {
 5492     if (Universe::heap() != NULL) {
 5493       if (CompressedOops::base() == NULL) {
 5494         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 5495       } else {
 5496         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
 5497       }
 5498     } else {
 5499       movptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
 5500     }
 5501   }
 5502 }
 5503 
 5504 #endif // _LP64
 5505 
 5506 #ifdef COMPILER2
 5507 // C2 compiled method's prolog code.
 5508 void MacroAssembler::verified_entry(Compile* C, int sp_inc) {
 5509   int framesize = C->output()->frame_size_in_bytes();
 5510   int bangsize = C->output()->bang_size_in_bytes();
 5511   bool fp_mode_24b = false;
 5512   int stack_bang_size = C->output()->need_stack_bang(bangsize) ? bangsize : 0;
 5513 
 5514   // WARNING: Initial instruction MUST be 5 bytes or longer so that
 5515   // NativeJump::patch_verified_entry will be able to patch out the entry
 5516   // code safely. The push to verify stack depth is ok at 5 bytes,
 5517   // the frame allocation can be either 3 or 6 bytes. So if we don't do
 5518   // stack bang then we must use the 6 byte frame allocation even if
 5519   // we have no frame. :-(
 5520   assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
 5521 
 5522   assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 5523   // Remove word for return addr
 5524   framesize -= wordSize;
 5525   stack_bang_size -= wordSize;
 5526 
 5527   // Calls to C2R adapters often do not accept exceptional returns.
 5528   // We require that their callers must bang for them.  But be careful, because
 5529   // some VM calls (such as call site linkage) can use several kilobytes of
 5530   // stack.  But the stack safety zone should account for that.
 5531   // See bugs 4446381, 4468289, 4497237.
 5532   if (stack_bang_size > 0) {

 5545     // Create frame
 5546     if (framesize) {
 5547       subptr(rsp, framesize);
 5548     }
 5549   } else {
 5550     // Create frame (force generation of a 4 byte immediate value)
 5551     subptr_imm32(rsp, framesize);
 5552 
 5553     // Save RBP register now.
 5554     framesize -= wordSize;
 5555     movptr(Address(rsp, framesize), rbp);
 5556     // Save caller's stack pointer into RBP if the frame pointer is preserved.
 5557     if (PreserveFramePointer) {
 5558       movptr(rbp, rsp);
 5559       if (framesize > 0) {
 5560         addptr(rbp, framesize);
 5561       }
 5562     }
 5563   }
 5564 
 5565   if (C->needs_stack_repair()) {
 5566     // Save stack increment just below the saved rbp (also account for fixed framesize and rbp)
 5567     assert((sp_inc & (StackAlignmentInBytes-1)) == 0, "stack increment not aligned");
 5568     movptr(Address(rsp, framesize - wordSize), sp_inc + framesize + wordSize);
 5569   }
 5570 
 5571   if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
 5572     framesize -= wordSize;
 5573     movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
 5574   }
 5575 
 5576 #ifndef _LP64
 5577   // If method sets FPU control word do it now
 5578   if (fp_mode_24b) {
 5579     fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
 5580   }
 5581   if (UseSSE >= 2 && VerifyFPU) {
 5582     verify_FPU(0, "FPU stack must be clean on entry");
 5583   }
 5584 #endif
 5585 
 5586 #ifdef ASSERT
 5587   if (VerifyStackAtCalls) {
 5588     Label L;
 5589     push(rax);
 5590     mov(rax, rsp);
 5591     andptr(rax, StackAlignmentInBytes-1);
 5592     cmpptr(rax, StackAlignmentInBytes-wordSize);
 5593     pop(rax);
 5594     jcc(Assembler::equal, L);
 5595     STOP("Stack is not properly aligned!");
 5596     bind(L);
 5597   }
 5598 #endif





 5599 }
 5600 #endif // COMPILER2
 5601 
 5602 #if COMPILER2_OR_JVMCI
 5603 
 5604 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 5605 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask) {
 5606   // cnt - number of qwords (8-byte words).
 5607   // base - start address, qword aligned.
 5608   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 5609   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 5610   if (use64byteVector) {
 5611     evpbroadcastq(xtmp, val, AVX_512bit);
 5612   } else if (MaxVectorSize >= 32) {
 5613     movdq(xtmp, val);
 5614     punpcklqdq(xtmp, xtmp);
 5615     vinserti128_high(xtmp, xtmp);
 5616   } else {
 5617     movdq(xtmp, val);
 5618     punpcklqdq(xtmp, xtmp);
 5619   }
 5620   jmp(L_zero_64_bytes);
 5621 
 5622   BIND(L_loop);
 5623   if (MaxVectorSize >= 32) {
 5624     fill64(base, 0, xtmp, use64byteVector);
 5625   } else {
 5626     movdqu(Address(base,  0), xtmp);
 5627     movdqu(Address(base, 16), xtmp);
 5628     movdqu(Address(base, 32), xtmp);
 5629     movdqu(Address(base, 48), xtmp);
 5630   }
 5631   addptr(base, 64);
 5632 
 5633   BIND(L_zero_64_bytes);
 5634   subptr(cnt, 8);
 5635   jccb(Assembler::greaterEqual, L_loop);
 5636 
 5637   // Copy trailing 64 bytes
 5638   if (use64byteVector) {
 5639     addptr(cnt, 8);
 5640     jccb(Assembler::equal, L_end);
 5641     fill64_masked(3, base, 0, xtmp, mask, cnt, val, true);
 5642     jmp(L_end);
 5643   } else {
 5644     addptr(cnt, 4);
 5645     jccb(Assembler::less, L_tail);
 5646     if (MaxVectorSize >= 32) {
 5647       vmovdqu(Address(base, 0), xtmp);
 5648     } else {
 5649       movdqu(Address(base,  0), xtmp);
 5650       movdqu(Address(base, 16), xtmp);
 5651     }
 5652   }
 5653   addptr(base, 32);
 5654   subptr(cnt, 4);
 5655 
 5656   BIND(L_tail);
 5657   addptr(cnt, 4);
 5658   jccb(Assembler::lessEqual, L_end);
 5659   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 5660     fill32_masked(3, base, 0, xtmp, mask, cnt, val);
 5661   } else {
 5662     decrement(cnt);
 5663 
 5664     BIND(L_sloop);
 5665     movq(Address(base, 0), xtmp);
 5666     addptr(base, 8);
 5667     decrement(cnt);
 5668     jccb(Assembler::greaterEqual, L_sloop);
 5669   }
 5670   BIND(L_end);
 5671 }
 5672 
 5673 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
 5674   assert(InlineTypeReturnedAsFields, "Inline types should never be returned as fields");
 5675   // An inline type might be returned. If fields are in registers we
 5676   // need to allocate an inline type instance and initialize it with
 5677   // the value of the fields.
 5678   Label skip;
 5679   // We only need a new buffered inline type if a new one is not returned
 5680   testptr(rax, 1);
 5681   jcc(Assembler::zero, skip);
 5682   int call_offset = -1;
 5683 
 5684 #ifdef _LP64
 5685   // The following code is similar to allocate_instance but has some slight differences,
 5686   // e.g. object size is always not zero, sometimes it's constant; storing klass ptr after
 5687   // allocating is not necessary if vk != NULL, etc. allocate_instance is not aware of these.
 5688   Label slow_case;
 5689   // 1. Try to allocate a new buffered inline instance either from TLAB or eden space
 5690   mov(rscratch1, rax); // save rax for slow_case since *_allocate may corrupt it when allocation failed
 5691   if (vk != NULL) {
 5692     // Called from C1, where the return type is statically known.
 5693     movptr(rbx, (intptr_t)vk->get_InlineKlass());
 5694     jint obj_size = vk->layout_helper();
 5695     assert(obj_size != Klass::_lh_neutral_value, "inline class in return type must have been resolved");
 5696     if (UseTLAB) {
 5697       tlab_allocate(r15_thread, rax, noreg, obj_size, r13, r14, slow_case);
 5698     } else {
 5699       eden_allocate(r15_thread, rax, noreg, obj_size, r13, slow_case);
 5700     }
 5701   } else {
 5702     // Call from interpreter. RAX contains ((the InlineKlass* of the return type) | 0x01)
 5703     mov(rbx, rax);
 5704     andptr(rbx, -2);
 5705     movl(r14, Address(rbx, Klass::layout_helper_offset()));
 5706     if (UseTLAB) {
 5707       tlab_allocate(r15_thread, rax, r14, 0, r13, r14, slow_case);
 5708     } else {
 5709       eden_allocate(r15_thread, rax, r14, 0, r13, slow_case);
 5710     }
 5711   }
 5712   if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) {
 5713     // 2. Initialize buffered inline instance header
 5714     Register buffer_obj = rax;
 5715     movptr(Address(buffer_obj, oopDesc::mark_offset_in_bytes()), (intptr_t)markWord::inline_type_prototype().value());
 5716     xorl(r13, r13);
 5717     store_klass_gap(buffer_obj, r13);
 5718     if (vk == NULL) {
 5719       // store_klass corrupts rbx(klass), so save it in r13 for later use (interpreter case only).
 5720       mov(r13, rbx);
 5721     }
 5722     Register tmp_store_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
 5723     store_klass(buffer_obj, rbx, tmp_store_klass);
 5724     // 3. Initialize its fields with an inline class specific handler
 5725     if (vk != NULL) {
 5726       call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint.
 5727     } else {
 5728       movptr(rbx, Address(r13, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 5729       movptr(rbx, Address(rbx, InlineKlass::pack_handler_offset()));
 5730       call(rbx);
 5731     }
 5732     jmp(skip);
 5733   }
 5734   bind(slow_case);
 5735   // We failed to allocate a new inline type, fall back to a runtime
 5736   // call. Some oop field may be live in some registers but we can't
 5737   // tell. That runtime call will take care of preserving them
 5738   // across a GC if there's one.
 5739   mov(rax, rscratch1);
 5740 #endif
 5741 
 5742   if (from_interpreter) {
 5743     super_call_VM_leaf(StubRoutines::store_inline_type_fields_to_buf());
 5744   } else {
 5745     call(RuntimeAddress(StubRoutines::store_inline_type_fields_to_buf()));
 5746     call_offset = offset();
 5747   }
 5748 
 5749   bind(skip);
 5750   return call_offset;
 5751 }
 5752 
 5753 // Move a value between registers/stack slots and update the reg_state
 5754 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
 5755   assert(from->is_valid() && to->is_valid(), "source and destination must be valid");
 5756   if (reg_state[to->value()] == reg_written) {
 5757     return true; // Already written
 5758   }
 5759   if (from != to && bt != T_VOID) {
 5760     if (reg_state[to->value()] == reg_readonly) {
 5761       return false; // Not yet writable
 5762     }
 5763     if (from->is_reg()) {
 5764       if (to->is_reg()) {
 5765         if (from->is_XMMRegister()) {
 5766           if (bt == T_DOUBLE) {
 5767             movdbl(to->as_XMMRegister(), from->as_XMMRegister());
 5768           } else {
 5769             assert(bt == T_FLOAT, "must be float");
 5770             movflt(to->as_XMMRegister(), from->as_XMMRegister());
 5771           }
 5772         } else {
 5773           movq(to->as_Register(), from->as_Register());
 5774         }
 5775       } else {
 5776         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 5777         Address to_addr = Address(rsp, st_off);
 5778         if (from->is_XMMRegister()) {
 5779           if (bt == T_DOUBLE) {
 5780             movdbl(to_addr, from->as_XMMRegister());
 5781           } else {
 5782             assert(bt == T_FLOAT, "must be float");
 5783             movflt(to_addr, from->as_XMMRegister());
 5784           }
 5785         } else {
 5786           movq(to_addr, from->as_Register());
 5787         }
 5788       }
 5789     } else {
 5790       Address from_addr = Address(rsp, from->reg2stack() * VMRegImpl::stack_slot_size + wordSize);
 5791       if (to->is_reg()) {
 5792         if (to->is_XMMRegister()) {
 5793           if (bt == T_DOUBLE) {
 5794             movdbl(to->as_XMMRegister(), from_addr);
 5795           } else {
 5796             assert(bt == T_FLOAT, "must be float");
 5797             movflt(to->as_XMMRegister(), from_addr);
 5798           }
 5799         } else {
 5800           movq(to->as_Register(), from_addr);
 5801         }
 5802       } else {
 5803         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 5804         movq(r13, from_addr);
 5805         movq(Address(rsp, st_off), r13);
 5806       }
 5807     }
 5808   }
 5809   // Update register states
 5810   reg_state[from->value()] = reg_writable;
 5811   reg_state[to->value()] = reg_written;
 5812   return true;
 5813 }
 5814 
 5815 // Calculate the extra stack space required for packing or unpacking inline
 5816 // args and adjust the stack pointer
 5817 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
 5818   // Two additional slots to account for return address
 5819   int sp_inc = (args_on_stack + 2) * VMRegImpl::stack_slot_size;
 5820   sp_inc = align_up(sp_inc, StackAlignmentInBytes);
 5821   // Save the return address, adjust the stack (make sure it is properly
 5822   // 16-byte aligned) and copy the return address to the new top of the stack.
 5823   // The stack will be repaired on return (see MacroAssembler::remove_frame).
 5824   assert(sp_inc > 0, "sanity");
 5825   pop(r13);
 5826   subptr(rsp, sp_inc);
 5827   push(r13);
 5828   return sp_inc;
 5829 }
 5830 
 5831 // Read all fields from an inline type buffer and store the field values in registers/stack slots.
 5832 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
 5833                                           VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
 5834                                           RegState reg_state[]) {
 5835   assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter");
 5836   assert(from->is_valid(), "source must be valid");
 5837   bool progress = false;
 5838 #ifdef ASSERT
 5839   const int start_offset = offset();
 5840 #endif
 5841 
 5842   Label L_null, L_notNull;
 5843   // Don't use r14 as tmp because it's used for spilling (see MacroAssembler::spill_reg_for)
 5844   Register tmp1 = r10;
 5845   Register tmp2 = r13;
 5846   Register fromReg = noreg;
 5847   ScalarizedInlineArgsStream stream(sig, sig_index, to, to_count, to_index, -1);
 5848   bool done = true;
 5849   bool mark_done = true;
 5850   VMReg toReg;
 5851   BasicType bt;
 5852   // Check if argument requires a null check
 5853   bool null_check = false;
 5854   VMReg nullCheckReg;
 5855   while (stream.next(nullCheckReg, bt)) {
 5856     if (sig->at(stream.sig_index())._offset == -1) {
 5857       null_check = true;
 5858       break;
 5859     }
 5860   }
 5861   stream.reset(sig_index, to_index);
 5862   while (stream.next(toReg, bt)) {
 5863     assert(toReg->is_valid(), "destination must be valid");
 5864     int idx = (int)toReg->value();
 5865     if (reg_state[idx] == reg_readonly) {
 5866       if (idx != from->value()) {
 5867         mark_done = false;
 5868       }
 5869       done = false;
 5870       continue;
 5871     } else if (reg_state[idx] == reg_written) {
 5872       continue;
 5873     }
 5874     assert(reg_state[idx] == reg_writable, "must be writable");
 5875     reg_state[idx] = reg_written;
 5876     progress = true;
 5877 
 5878     if (fromReg == noreg) {
 5879       if (from->is_reg()) {
 5880         fromReg = from->as_Register();
 5881       } else {
 5882         int st_off = from->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 5883         movq(tmp1, Address(rsp, st_off));
 5884         fromReg = tmp1;
 5885       }
 5886       if (null_check) {
 5887         // Nullable inline type argument, emit null check
 5888         testptr(fromReg, fromReg);
 5889         jcc(Assembler::zero, L_null);
 5890       }
 5891     }
 5892     int off = sig->at(stream.sig_index())._offset;
 5893     if (off == -1) {
 5894       assert(null_check, "Missing null check at");
 5895       if (toReg->is_stack()) {
 5896         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 5897         movq(Address(rsp, st_off), 1);
 5898       } else {
 5899         movq(toReg->as_Register(), 1);
 5900       }
 5901       continue;
 5902     }
 5903     assert(off > 0, "offset in object should be positive");
 5904     Address fromAddr = Address(fromReg, off);
 5905     if (!toReg->is_XMMRegister()) {
 5906       Register dst = toReg->is_stack() ? tmp2 : toReg->as_Register();
 5907       if (is_reference_type(bt)) {
 5908         load_heap_oop(dst, fromAddr);
 5909       } else {
 5910         bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN);
 5911         load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed);
 5912       }
 5913       if (toReg->is_stack()) {
 5914         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 5915         movq(Address(rsp, st_off), dst);
 5916       }
 5917     } else if (bt == T_DOUBLE) {
 5918       movdbl(toReg->as_XMMRegister(), fromAddr);
 5919     } else {
 5920       assert(bt == T_FLOAT, "must be float");
 5921       movflt(toReg->as_XMMRegister(), fromAddr);
 5922     }
 5923   }
 5924   if (progress && null_check) {
 5925     if (done) {
 5926       jmp(L_notNull);
 5927       bind(L_null);
 5928       // Set IsInit field to zero to signal that the argument is null.
 5929       // Also set all oop fields to zero to make the GC happy.
 5930       stream.reset(sig_index, to_index);
 5931       while (stream.next(toReg, bt)) {
 5932         if (sig->at(stream.sig_index())._offset == -1 ||
 5933             bt == T_OBJECT || bt == T_ARRAY) {
 5934           if (toReg->is_stack()) {
 5935             int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 5936             movq(Address(rsp, st_off), 0);
 5937           } else {
 5938             xorq(toReg->as_Register(), toReg->as_Register());
 5939           }
 5940         }
 5941       }
 5942       bind(L_notNull);
 5943     } else {
 5944       bind(L_null);
 5945     }
 5946   }
 5947 
 5948   sig_index = stream.sig_index();
 5949   to_index = stream.regs_index();
 5950 
 5951   if (mark_done && reg_state[from->value()] != reg_written) {
 5952     // This is okay because no one else will write to that slot
 5953     reg_state[from->value()] = reg_writable;
 5954   }
 5955   from_index--;
 5956   assert(progress || (start_offset == offset()), "should not emit code");
 5957   return done;
 5958 }
 5959 
 5960 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
 5961                                         VMRegPair* from, int from_count, int& from_index, VMReg to,
 5962                                         RegState reg_state[], Register val_array) {
 5963   assert(sig->at(sig_index)._bt == T_PRIMITIVE_OBJECT, "should be at end delimiter");
 5964   assert(to->is_valid(), "destination must be valid");
 5965 
 5966   if (reg_state[to->value()] == reg_written) {
 5967     skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
 5968     return true; // Already written
 5969   }
 5970 
 5971   // TODO 8284443 Isn't it an issue if below code uses r14 as tmp when it contains a spilled value?
 5972   // Be careful with r14 because it's used for spilling (see MacroAssembler::spill_reg_for).
 5973   Register val_obj_tmp = r11;
 5974   Register from_reg_tmp = r14;
 5975   Register tmp1 = r10;
 5976   Register tmp2 = r13;
 5977   Register tmp3 = rbx;
 5978   Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register();
 5979 
 5980   assert_different_registers(val_obj_tmp, from_reg_tmp, tmp1, tmp2, tmp3, val_array);
 5981 
 5982   if (reg_state[to->value()] == reg_readonly) {
 5983     if (!is_reg_in_unpacked_fields(sig, sig_index, to, from, from_count, from_index)) {
 5984       skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
 5985       return false; // Not yet writable
 5986     }
 5987     val_obj = val_obj_tmp;
 5988   }
 5989 
 5990   int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_PRIMITIVE_OBJECT);
 5991   load_heap_oop(val_obj, Address(val_array, index));
 5992 
 5993   ScalarizedInlineArgsStream stream(sig, sig_index, from, from_count, from_index);
 5994   VMReg fromReg;
 5995   BasicType bt;
 5996   Label L_null;
 5997   while (stream.next(fromReg, bt)) {
 5998     assert(fromReg->is_valid(), "source must be valid");
 5999     reg_state[fromReg->value()] = reg_writable;
 6000 
 6001     int off = sig->at(stream.sig_index())._offset;
 6002     if (off == -1) {
 6003       // Nullable inline type argument, emit null check
 6004       Label L_notNull;
 6005       if (fromReg->is_stack()) {
 6006         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6007         testb(Address(rsp, ld_off), 1);
 6008       } else {
 6009         testb(fromReg->as_Register(), 1);
 6010       }
 6011       jcc(Assembler::notZero, L_notNull);
 6012       movptr(val_obj, 0);
 6013       jmp(L_null);
 6014       bind(L_notNull);
 6015       continue;
 6016     }
 6017 
 6018     assert(off > 0, "offset in object should be positive");
 6019     size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
 6020 
 6021     Address dst(val_obj, off);
 6022     if (!fromReg->is_XMMRegister()) {
 6023       Register src;
 6024       if (fromReg->is_stack()) {
 6025         src = from_reg_tmp;
 6026         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6027         load_sized_value(src, Address(rsp, ld_off), size_in_bytes, /* is_signed */ false);
 6028       } else {
 6029         src = fromReg->as_Register();
 6030       }
 6031       assert_different_registers(dst.base(), src, tmp1, tmp2, tmp3, val_array);
 6032       if (is_reference_type(bt)) {
 6033         store_heap_oop(dst, src, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
 6034       } else {
 6035         store_sized_value(dst, src, size_in_bytes);
 6036       }
 6037     } else if (bt == T_DOUBLE) {
 6038       movdbl(dst, fromReg->as_XMMRegister());
 6039     } else {
 6040       assert(bt == T_FLOAT, "must be float");
 6041       movflt(dst, fromReg->as_XMMRegister());
 6042     }
 6043   }
 6044   bind(L_null);
 6045   sig_index = stream.sig_index();
 6046   from_index = stream.regs_index();
 6047 
 6048   assert(reg_state[to->value()] == reg_writable, "must have already been read");
 6049   bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state);
 6050   assert(success, "to register must be writeable");
 6051   return true;
 6052 }
 6053 
 6054 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
 6055   return reg->is_XMMRegister() ? xmm8->as_VMReg() : r14->as_VMReg();
 6056 }
 6057 
 6058 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
 6059   assert((initial_framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 6060   if (needs_stack_repair) {
 6061     movq(rbp, Address(rsp, initial_framesize));
 6062     // The stack increment resides just below the saved rbp
 6063     addq(rsp, Address(rsp, initial_framesize - wordSize));
 6064   } else {
 6065     if (initial_framesize > 0) {
 6066       addq(rsp, initial_framesize);
 6067     }
 6068     pop(rbp);
 6069   }
 6070 }
 6071 
 6072 // Clearing constant sized memory using YMM/ZMM registers.
 6073 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6074   assert(UseAVX > 2 && VM_Version::supports_avx512vlbw(), "");
 6075   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 6076 
 6077   int vector64_count = (cnt & (~0x7)) >> 3;
 6078   cnt = cnt & 0x7;
 6079 
 6080   // 64 byte initialization loop.
 6081   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 6082   for (int i = 0; i < vector64_count; i++) {
 6083     fill64(base, i * 64, xtmp, use64byteVector);
 6084   }
 6085 
 6086   // Clear remaining 64 byte tail.
 6087   int disp = vector64_count * 64;
 6088   if (cnt) {
 6089     switch (cnt) {
 6090       case 1:
 6091         movq(Address(base, disp), xtmp);

 6123         break;
 6124       case 7:
 6125         if (use64byteVector) {
 6126           movl(rtmp, 0x7F);
 6127           kmovwl(mask, rtmp);
 6128           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, Assembler::AVX_512bit);
 6129         } else {
 6130           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, Assembler::AVX_256bit);
 6131           movl(rtmp, 0x7);
 6132           kmovwl(mask, rtmp);
 6133           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, Assembler::AVX_256bit);
 6134         }
 6135         break;
 6136       default:
 6137         fatal("Unexpected length : %d\n",cnt);
 6138         break;
 6139     }
 6140   }
 6141 }
 6142 
 6143 void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp,
 6144                                bool is_large, bool word_copy_only, KRegister mask) {
 6145   // cnt      - number of qwords (8-byte words).
 6146   // base     - start address, qword aligned.
 6147   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 6148   assert(base==rdi, "base register must be edi for rep stos");
 6149   assert(val==rax,   "val register must be eax for rep stos");
 6150   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 6151   assert(InitArrayShortSize % BytesPerLong == 0,
 6152     "InitArrayShortSize should be the multiple of BytesPerLong");
 6153 
 6154   Label DONE;



 6155 
 6156   if (!is_large) {
 6157     Label LOOP, LONG;
 6158     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 6159     jccb(Assembler::greater, LONG);
 6160 
 6161     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6162 
 6163     decrement(cnt);
 6164     jccb(Assembler::negative, DONE); // Zero length
 6165 
 6166     // Use individual pointer-sized stores for small counts:
 6167     BIND(LOOP);
 6168     movptr(Address(base, cnt, Address::times_ptr), val);
 6169     decrement(cnt);
 6170     jccb(Assembler::greaterEqual, LOOP);
 6171     jmpb(DONE);
 6172 
 6173     BIND(LONG);
 6174   }
 6175 
 6176   // Use longer rep-prefixed ops for non-small counts:
 6177   if (UseFastStosb && !word_copy_only) {
 6178     shlptr(cnt, 3); // convert to number of bytes
 6179     rep_stosb();
 6180   } else if (UseXMMForObjInit) {
 6181     xmm_clear_mem(base, cnt, val, xtmp, mask);
 6182   } else {
 6183     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6184     rep_stos();
 6185   }
 6186 
 6187   BIND(DONE);
 6188 }
 6189 
 6190 #endif //COMPILER2_OR_JVMCI
 6191 
 6192 
 6193 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 6194                                    Register to, Register value, Register count,
 6195                                    Register rtmp, XMMRegister xtmp) {
 6196   ShortBranchVerifier sbv(this);
 6197   assert_different_registers(to, value, count, rtmp);
 6198   Label L_exit;
 6199   Label L_fill_2_bytes, L_fill_4_bytes;
 6200 
 6201 #if defined(COMPILER2) && defined(_LP64)
< prev index next >