< prev index next >

src/hotspot/cpu/x86/macroAssembler_x86.cpp

Print this page

   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "code/compiledIC.hpp"
   29 #include "compiler/compiler_globals.hpp"
   30 #include "compiler/disassembler.hpp"

   31 #include "crc32c.h"
   32 #include "gc/shared/barrierSet.hpp"
   33 #include "gc/shared/barrierSetAssembler.hpp"
   34 #include "gc/shared/collectedHeap.inline.hpp"
   35 #include "gc/shared/tlab_globals.hpp"
   36 #include "interpreter/bytecodeHistogram.hpp"
   37 #include "interpreter/interpreter.hpp"
   38 #include "jvm.h"
   39 #include "memory/resourceArea.hpp"
   40 #include "memory/universe.hpp"
   41 #include "oops/accessDecorators.hpp"
   42 #include "oops/compressedKlass.inline.hpp"
   43 #include "oops/compressedOops.inline.hpp"
   44 #include "oops/klass.inline.hpp"

   45 #include "prims/methodHandles.hpp"
   46 #include "runtime/continuation.hpp"
   47 #include "runtime/interfaceSupport.inline.hpp"
   48 #include "runtime/javaThread.hpp"
   49 #include "runtime/jniHandles.hpp"
   50 #include "runtime/objectMonitor.hpp"
   51 #include "runtime/os.hpp"
   52 #include "runtime/safepoint.hpp"
   53 #include "runtime/safepointMechanism.hpp"
   54 #include "runtime/sharedRuntime.hpp"

   55 #include "runtime/stubRoutines.hpp"
   56 #include "utilities/checkedCast.hpp"
   57 #include "utilities/macros.hpp"




   58 
   59 #ifdef PRODUCT
   60 #define BLOCK_COMMENT(str) /* nothing */
   61 #define STOP(error) stop(error)
   62 #else
   63 #define BLOCK_COMMENT(str) block_comment(str)
   64 #define STOP(error) block_comment(error); stop(error)
   65 #endif
   66 
   67 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   68 
   69 #ifdef ASSERT
   70 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   71 #endif
   72 
   73 static const Assembler::Condition reverse[] = {
   74     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   75     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   76     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   77     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,

 1701 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1702   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1703   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1704   pass_arg2(this, arg_2);
 1705   pass_arg1(this, arg_1);
 1706   pass_arg0(this, arg_0);
 1707   call_VM_leaf(entry_point, 3);
 1708 }
 1709 
 1710 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1711   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1712   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1713   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1714   pass_arg3(this, arg_3);
 1715   pass_arg2(this, arg_2);
 1716   pass_arg1(this, arg_1);
 1717   pass_arg0(this, arg_0);
 1718   call_VM_leaf(entry_point, 3);
 1719 }
 1720 




 1721 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1722   pass_arg0(this, arg_0);
 1723   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1724 }
 1725 
 1726 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1727   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1728   pass_arg1(this, arg_1);
 1729   pass_arg0(this, arg_0);
 1730   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1731 }
 1732 
 1733 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1734   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1735   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1736   pass_arg2(this, arg_2);
 1737   pass_arg1(this, arg_1);
 1738   pass_arg0(this, arg_0);
 1739   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 1740 }

 2885     lea(rscratch, src);
 2886     Assembler::mulss(dst, Address(rscratch, 0));
 2887   }
 2888 }
 2889 
 2890 void MacroAssembler::null_check(Register reg, int offset) {
 2891   if (needs_explicit_null_check(offset)) {
 2892     // provoke OS null exception if reg is null by
 2893     // accessing M[reg] w/o changing any (non-CC) registers
 2894     // NOTE: cmpl is plenty here to provoke a segv
 2895     cmpptr(rax, Address(reg, 0));
 2896     // Note: should probably use testl(rax, Address(reg, 0));
 2897     //       may be shorter code (however, this version of
 2898     //       testl needs to be implemented first)
 2899   } else {
 2900     // nothing to do, (later) access of M[reg + offset]
 2901     // will provoke OS null exception if reg is null
 2902   }
 2903 }
 2904 





























































































































 2905 void MacroAssembler::os_breakpoint() {
 2906   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 2907   // (e.g., MSVC can't call ps() otherwise)
 2908   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 2909 }
 2910 
 2911 void MacroAssembler::unimplemented(const char* what) {
 2912   const char* buf = nullptr;
 2913   {
 2914     ResourceMark rm;
 2915     stringStream ss;
 2916     ss.print("unimplemented: %s", what);
 2917     buf = code_string(ss.as_string());
 2918   }
 2919   stop(buf);
 2920 }
 2921 
 2922 #ifdef _LP64
 2923 #define XSTATE_BV 0x200
 2924 #endif

 4049 }
 4050 
 4051 // C++ bool manipulation
 4052 void MacroAssembler::testbool(Register dst) {
 4053   if(sizeof(bool) == 1)
 4054     testb(dst, 0xff);
 4055   else if(sizeof(bool) == 2) {
 4056     // testw implementation needed for two byte bools
 4057     ShouldNotReachHere();
 4058   } else if(sizeof(bool) == 4)
 4059     testl(dst, dst);
 4060   else
 4061     // unsupported
 4062     ShouldNotReachHere();
 4063 }
 4064 
 4065 void MacroAssembler::testptr(Register dst, Register src) {
 4066   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 4067 }
 4068 


















































































































 4069 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 4070 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 4071                                    Register var_size_in_bytes,
 4072                                    int con_size_in_bytes,
 4073                                    Register t1,
 4074                                    Register t2,
 4075                                    Label& slow_case) {
 4076   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4077   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 4078 }
 4079 
 4080 RegSet MacroAssembler::call_clobbered_gp_registers() {
 4081   RegSet regs;
 4082 #ifdef _LP64
 4083   regs += RegSet::of(rax, rcx, rdx);
 4084 #ifndef WINDOWS
 4085   regs += RegSet::of(rsi, rdi);
 4086 #endif
 4087   regs += RegSet::range(r8, r11);
 4088 #else

 4302     // clear topmost word (no jump would be needed if conditional assignment worked here)
 4303     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 4304     // index could be 0 now, must check again
 4305     jcc(Assembler::zero, done);
 4306     bind(even);
 4307   }
 4308 #endif // !_LP64
 4309   // initialize remaining object fields: index is a multiple of 2 now
 4310   {
 4311     Label loop;
 4312     bind(loop);
 4313     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 4314     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 4315     decrement(index);
 4316     jcc(Assembler::notZero, loop);
 4317   }
 4318 
 4319   bind(done);
 4320 }
 4321 


















































 4322 // Look up the method for a megamorphic invokeinterface call.
 4323 // The target method is determined by <intf_klass, itable_index>.
 4324 // The receiver klass is in recv_klass.
 4325 // On success, the result will be in method_result, and execution falls through.
 4326 // On failure, execution transfers to the given label.
 4327 void MacroAssembler::lookup_interface_method(Register recv_klass,
 4328                                              Register intf_klass,
 4329                                              RegisterOrConstant itable_index,
 4330                                              Register method_result,
 4331                                              Register scan_temp,
 4332                                              Label& L_no_such_interface,
 4333                                              bool return_method) {
 4334   assert_different_registers(recv_klass, intf_klass, scan_temp);
 4335   assert_different_registers(method_result, intf_klass, scan_temp);
 4336   assert(recv_klass != method_result || !return_method,
 4337          "recv_klass can be destroyed when method isn't needed");
 4338 
 4339   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 4340          "caller must use same register for non-constant itable index as for method");
 4341 

 5103   } else {
 5104     Label L;
 5105     jccb(negate_condition(cc), L);
 5106     movl(dst, src);
 5107     bind(L);
 5108   }
 5109 }
 5110 
 5111 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 5112   if (VM_Version::supports_cmov()) {
 5113     cmovl(cc, dst, src);
 5114   } else {
 5115     Label L;
 5116     jccb(negate_condition(cc), L);
 5117     movl(dst, src);
 5118     bind(L);
 5119   }
 5120 }
 5121 
 5122 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 5123   if (!VerifyOops) return;




 5124 
 5125   BLOCK_COMMENT("verify_oop {");
 5126 #ifdef _LP64
 5127   push(rscratch1);
 5128 #endif
 5129   push(rax);                          // save rax
 5130   push(reg);                          // pass register argument
 5131 
 5132   // Pass register number to verify_oop_subroutine
 5133   const char* b = nullptr;
 5134   {
 5135     ResourceMark rm;
 5136     stringStream ss;
 5137     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 5138     b = code_string(ss.as_string());
 5139   }
 5140   ExternalAddress buffer((address) b);
 5141   pushptr(buffer.addr(), rscratch1);
 5142 
 5143   // call indirectly to solve generation ordering problem

 5165   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 5166   int stackElementSize = Interpreter::stackElementSize;
 5167   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 5168 #ifdef ASSERT
 5169   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 5170   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 5171 #endif
 5172   Register             scale_reg    = noreg;
 5173   Address::ScaleFactor scale_factor = Address::no_scale;
 5174   if (arg_slot.is_constant()) {
 5175     offset += arg_slot.as_constant() * stackElementSize;
 5176   } else {
 5177     scale_reg    = arg_slot.as_register();
 5178     scale_factor = Address::times(stackElementSize);
 5179   }
 5180   offset += wordSize;           // return PC is on stack
 5181   return Address(rsp, scale_reg, scale_factor, offset);
 5182 }
 5183 
 5184 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 5185   if (!VerifyOops) return;




 5186 
 5187 #ifdef _LP64
 5188   push(rscratch1);
 5189 #endif
 5190   push(rax); // save rax,
 5191   // addr may contain rsp so we will have to adjust it based on the push
 5192   // we just did (and on 64 bit we do two pushes)
 5193   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 5194   // stores rax into addr which is backwards of what was intended.
 5195   if (addr.uses(rsp)) {
 5196     lea(rax, addr);
 5197     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
 5198   } else {
 5199     pushptr(addr);
 5200   }
 5201 
 5202   // Pass register number to verify_oop_subroutine
 5203   const char* b = nullptr;
 5204   {
 5205     ResourceMark rm;

 5652 
 5653 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 5654   // get mirror
 5655   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 5656   load_method_holder(mirror, method);
 5657   movptr(mirror, Address(mirror, mirror_offset));
 5658   resolve_oop_handle(mirror, tmp);
 5659 }
 5660 
 5661 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 5662   load_method_holder(rresult, rmethod);
 5663   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 5664 }
 5665 
 5666 void MacroAssembler::load_method_holder(Register holder, Register method) {
 5667   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 5668   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 5669   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
 5670 }
 5671 








 5672 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 5673   assert_different_registers(src, tmp);
 5674   assert_different_registers(dst, tmp);
 5675 #ifdef _LP64
 5676   if (UseCompressedClassPointers) {
 5677     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5678     decode_klass_not_null(dst, tmp);
 5679   } else
 5680 #endif
 5681     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));





 5682 }
 5683 
 5684 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 5685   assert_different_registers(src, tmp);
 5686   assert_different_registers(dst, tmp);
 5687 #ifdef _LP64
 5688   if (UseCompressedClassPointers) {
 5689     encode_klass_not_null(src, tmp);
 5690     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5691   } else
 5692 #endif
 5693     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 5694 }
 5695 
 5696 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 5697                                     Register tmp1, Register thread_tmp) {
 5698   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5699   decorators = AccessInternal::decorator_fixup(decorators, type);
 5700   bool as_raw = (decorators & AS_RAW) != 0;
 5701   if (as_raw) {
 5702     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5703   } else {
 5704     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 5705   }
 5706 }
 5707 
 5708 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
 5709                                      Register tmp1, Register tmp2, Register tmp3) {
 5710   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 5711   decorators = AccessInternal::decorator_fixup(decorators, type);
 5712   bool as_raw = (decorators & AS_RAW) != 0;
 5713   if (as_raw) {
 5714     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5715   } else {
 5716     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 5717   }
 5718 }
 5719 








































 5720 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 5721                                    Register thread_tmp, DecoratorSet decorators) {
 5722   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 5723 }
 5724 
 5725 // Doesn't do verification, generates fixed size code
 5726 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 5727                                             Register thread_tmp, DecoratorSet decorators) {
 5728   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 5729 }
 5730 
 5731 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
 5732                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 5733   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
 5734 }
 5735 
 5736 // Used for storing nulls.
 5737 void MacroAssembler::store_heap_oop_null(Address dst) {
 5738   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 5739 }

 6039 
 6040 void MacroAssembler::reinit_heapbase() {
 6041   if (UseCompressedOops) {
 6042     if (Universe::heap() != nullptr) {
 6043       if (CompressedOops::base() == nullptr) {
 6044         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 6045       } else {
 6046         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
 6047       }
 6048     } else {
 6049       movptr(r12_heapbase, ExternalAddress(CompressedOops::ptrs_base_addr()));
 6050     }
 6051   }
 6052 }
 6053 
 6054 #endif // _LP64
 6055 
 6056 #if COMPILER2_OR_JVMCI
 6057 
 6058 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 6059 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6060   // cnt - number of qwords (8-byte words).
 6061   // base - start address, qword aligned.
 6062   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 6063   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 6064   if (use64byteVector) {
 6065     vpxor(xtmp, xtmp, xtmp, AVX_512bit);
 6066   } else if (MaxVectorSize >= 32) {
 6067     vpxor(xtmp, xtmp, xtmp, AVX_256bit);


 6068   } else {
 6069     pxor(xtmp, xtmp);

 6070   }
 6071   jmp(L_zero_64_bytes);
 6072 
 6073   BIND(L_loop);
 6074   if (MaxVectorSize >= 32) {
 6075     fill64(base, 0, xtmp, use64byteVector);
 6076   } else {
 6077     movdqu(Address(base,  0), xtmp);
 6078     movdqu(Address(base, 16), xtmp);
 6079     movdqu(Address(base, 32), xtmp);
 6080     movdqu(Address(base, 48), xtmp);
 6081   }
 6082   addptr(base, 64);
 6083 
 6084   BIND(L_zero_64_bytes);
 6085   subptr(cnt, 8);
 6086   jccb(Assembler::greaterEqual, L_loop);
 6087 
 6088   // Copy trailing 64 bytes
 6089   if (use64byteVector) {
 6090     addptr(cnt, 8);
 6091     jccb(Assembler::equal, L_end);
 6092     fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true);
 6093     jmp(L_end);
 6094   } else {
 6095     addptr(cnt, 4);
 6096     jccb(Assembler::less, L_tail);
 6097     if (MaxVectorSize >= 32) {
 6098       vmovdqu(Address(base, 0), xtmp);
 6099     } else {
 6100       movdqu(Address(base,  0), xtmp);
 6101       movdqu(Address(base, 16), xtmp);
 6102     }
 6103   }
 6104   addptr(base, 32);
 6105   subptr(cnt, 4);
 6106 
 6107   BIND(L_tail);
 6108   addptr(cnt, 4);
 6109   jccb(Assembler::lessEqual, L_end);
 6110   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 6111     fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp);
 6112   } else {
 6113     decrement(cnt);
 6114 
 6115     BIND(L_sloop);
 6116     movq(Address(base, 0), xtmp);
 6117     addptr(base, 8);
 6118     decrement(cnt);
 6119     jccb(Assembler::greaterEqual, L_sloop);
 6120   }
 6121   BIND(L_end);
 6122 }
 6123 
















































































































































































































































































































































































































 6124 // Clearing constant sized memory using YMM/ZMM registers.
 6125 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6126   assert(UseAVX > 2 && VM_Version::supports_avx512vl(), "");
 6127   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 6128 
 6129   int vector64_count = (cnt & (~0x7)) >> 3;
 6130   cnt = cnt & 0x7;
 6131   const int fill64_per_loop = 4;
 6132   const int max_unrolled_fill64 = 8;
 6133 
 6134   // 64 byte initialization loop.
 6135   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 6136   int start64 = 0;
 6137   if (vector64_count > max_unrolled_fill64) {
 6138     Label LOOP;
 6139     Register index = rtmp;
 6140 
 6141     start64 = vector64_count - (vector64_count % fill64_per_loop);
 6142 
 6143     movl(index, 0);

 6193         break;
 6194       case 7:
 6195         if (use64byteVector) {
 6196           movl(rtmp, 0x7F);
 6197           kmovwl(mask, rtmp);
 6198           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6199         } else {
 6200           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6201           movl(rtmp, 0x7);
 6202           kmovwl(mask, rtmp);
 6203           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
 6204         }
 6205         break;
 6206       default:
 6207         fatal("Unexpected length : %d\n",cnt);
 6208         break;
 6209     }
 6210   }
 6211 }
 6212 
 6213 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp,
 6214                                bool is_large, KRegister mask) {
 6215   // cnt      - number of qwords (8-byte words).
 6216   // base     - start address, qword aligned.
 6217   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 6218   assert(base==rdi, "base register must be edi for rep stos");
 6219   assert(tmp==rax,   "tmp register must be eax for rep stos");
 6220   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 6221   assert(InitArrayShortSize % BytesPerLong == 0,
 6222     "InitArrayShortSize should be the multiple of BytesPerLong");
 6223 
 6224   Label DONE;
 6225   if (!is_large || !UseXMMForObjInit) {
 6226     xorptr(tmp, tmp);
 6227   }
 6228 
 6229   if (!is_large) {
 6230     Label LOOP, LONG;
 6231     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 6232     jccb(Assembler::greater, LONG);
 6233 
 6234     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6235 
 6236     decrement(cnt);
 6237     jccb(Assembler::negative, DONE); // Zero length
 6238 
 6239     // Use individual pointer-sized stores for small counts:
 6240     BIND(LOOP);
 6241     movptr(Address(base, cnt, Address::times_ptr), tmp);
 6242     decrement(cnt);
 6243     jccb(Assembler::greaterEqual, LOOP);
 6244     jmpb(DONE);
 6245 
 6246     BIND(LONG);
 6247   }
 6248 
 6249   // Use longer rep-prefixed ops for non-small counts:
 6250   if (UseFastStosb) {
 6251     shlptr(cnt, 3); // convert to number of bytes
 6252     rep_stosb();
 6253   } else if (UseXMMForObjInit) {
 6254     xmm_clear_mem(base, cnt, tmp, xtmp, mask);
 6255   } else {
 6256     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6257     rep_stos();
 6258   }
 6259 
 6260   BIND(DONE);
 6261 }
 6262 
 6263 #endif //COMPILER2_OR_JVMCI
 6264 
 6265 
 6266 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 6267                                    Register to, Register value, Register count,
 6268                                    Register rtmp, XMMRegister xtmp) {
 6269   ShortBranchVerifier sbv(this);
 6270   assert_different_registers(to, value, count, rtmp);
 6271   Label L_exit;
 6272   Label L_fill_2_bytes, L_fill_4_bytes;
 6273 
 6274 #if defined(COMPILER2) && defined(_LP64)

10284 
10285   // Load top.
10286   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10287 
10288   // Check if the lock-stack is full.
10289   cmpl(top, LockStack::end_offset());
10290   jcc(Assembler::greaterEqual, slow);
10291 
10292   // Check for recursion.
10293   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
10294   jcc(Assembler::equal, push);
10295 
10296   // Check header for monitor (0b10).
10297   testptr(reg_rax, markWord::monitor_value);
10298   jcc(Assembler::notZero, slow);
10299 
10300   // Try to lock. Transition lock bits 0b01 => 0b00
10301   movptr(tmp, reg_rax);
10302   andptr(tmp, ~(int32_t)markWord::unlocked_value);
10303   orptr(reg_rax, markWord::unlocked_value);




10304   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
10305   jcc(Assembler::notEqual, slow);
10306 
10307   // Restore top, CAS clobbers register.
10308   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
10309 
10310   bind(push);
10311   // After successful lock, push object on lock-stack.
10312   movptr(Address(thread, top), obj);
10313   incrementl(top, oopSize);
10314   movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
10315 }
10316 
10317 // Implements lightweight-unlocking.
10318 //
10319 // obj: the object to be unlocked
10320 // reg_rax: rax
10321 // thread: the thread
10322 // tmp: a temporary register
10323 //

   11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
   12  * version 2 for more details (a copy is included in the LICENSE file that
   13  * accompanied this code).
   14  *
   15  * You should have received a copy of the GNU General Public License version
   16  * 2 along with this work; if not, write to the Free Software Foundation,
   17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
   18  *
   19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
   20  * or visit www.oracle.com if you need additional information or have any
   21  * questions.
   22  *
   23  */
   24 
   25 #include "precompiled.hpp"
   26 #include "asm/assembler.hpp"
   27 #include "asm/assembler.inline.hpp"
   28 #include "code/compiledIC.hpp"
   29 #include "compiler/compiler_globals.hpp"
   30 #include "compiler/disassembler.hpp"
   31 #include "ci/ciInlineKlass.hpp"
   32 #include "crc32c.h"
   33 #include "gc/shared/barrierSet.hpp"
   34 #include "gc/shared/barrierSetAssembler.hpp"
   35 #include "gc/shared/collectedHeap.inline.hpp"
   36 #include "gc/shared/tlab_globals.hpp"
   37 #include "interpreter/bytecodeHistogram.hpp"
   38 #include "interpreter/interpreter.hpp"
   39 #include "jvm.h"
   40 #include "memory/resourceArea.hpp"
   41 #include "memory/universe.hpp"
   42 #include "oops/accessDecorators.hpp"
   43 #include "oops/compressedKlass.inline.hpp"
   44 #include "oops/compressedOops.inline.hpp"
   45 #include "oops/klass.inline.hpp"
   46 #include "oops/resolvedFieldEntry.hpp"
   47 #include "prims/methodHandles.hpp"
   48 #include "runtime/continuation.hpp"
   49 #include "runtime/interfaceSupport.inline.hpp"
   50 #include "runtime/javaThread.hpp"
   51 #include "runtime/jniHandles.hpp"
   52 #include "runtime/objectMonitor.hpp"
   53 #include "runtime/os.hpp"
   54 #include "runtime/safepoint.hpp"
   55 #include "runtime/safepointMechanism.hpp"
   56 #include "runtime/sharedRuntime.hpp"
   57 #include "runtime/signature_cc.hpp"
   58 #include "runtime/stubRoutines.hpp"
   59 #include "utilities/checkedCast.hpp"
   60 #include "utilities/macros.hpp"
   61 #include "vmreg_x86.inline.hpp"
   62 #ifdef COMPILER2
   63 #include "opto/output.hpp"
   64 #endif
   65 
   66 #ifdef PRODUCT
   67 #define BLOCK_COMMENT(str) /* nothing */
   68 #define STOP(error) stop(error)
   69 #else
   70 #define BLOCK_COMMENT(str) block_comment(str)
   71 #define STOP(error) block_comment(error); stop(error)
   72 #endif
   73 
   74 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
   75 
   76 #ifdef ASSERT
   77 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
   78 #endif
   79 
   80 static const Assembler::Condition reverse[] = {
   81     Assembler::noOverflow     /* overflow      = 0x0 */ ,
   82     Assembler::overflow       /* noOverflow    = 0x1 */ ,
   83     Assembler::aboveEqual     /* carrySet      = 0x2, below         = 0x2 */ ,
   84     Assembler::below          /* aboveEqual    = 0x3, carryClear    = 0x3 */ ,

 1708 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1709   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1710   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1711   pass_arg2(this, arg_2);
 1712   pass_arg1(this, arg_1);
 1713   pass_arg0(this, arg_0);
 1714   call_VM_leaf(entry_point, 3);
 1715 }
 1716 
 1717 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 1718   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3));
 1719   LP64_ONLY(assert_different_registers(arg_1, c_rarg2, c_rarg3));
 1720   LP64_ONLY(assert_different_registers(arg_2, c_rarg3));
 1721   pass_arg3(this, arg_3);
 1722   pass_arg2(this, arg_2);
 1723   pass_arg1(this, arg_1);
 1724   pass_arg0(this, arg_0);
 1725   call_VM_leaf(entry_point, 3);
 1726 }
 1727 
 1728 void MacroAssembler::super_call_VM_leaf(address entry_point) {
 1729   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1730 }
 1731 
 1732 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 1733   pass_arg0(this, arg_0);
 1734   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 1735 }
 1736 
 1737 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 1738   LP64_ONLY(assert_different_registers(arg_0, c_rarg1));
 1739   pass_arg1(this, arg_1);
 1740   pass_arg0(this, arg_0);
 1741   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 1742 }
 1743 
 1744 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 1745   LP64_ONLY(assert_different_registers(arg_0, c_rarg1, c_rarg2));
 1746   LP64_ONLY(assert_different_registers(arg_1, c_rarg2));
 1747   pass_arg2(this, arg_2);
 1748   pass_arg1(this, arg_1);
 1749   pass_arg0(this, arg_0);
 1750   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 1751 }

 2896     lea(rscratch, src);
 2897     Assembler::mulss(dst, Address(rscratch, 0));
 2898   }
 2899 }
 2900 
 2901 void MacroAssembler::null_check(Register reg, int offset) {
 2902   if (needs_explicit_null_check(offset)) {
 2903     // provoke OS null exception if reg is null by
 2904     // accessing M[reg] w/o changing any (non-CC) registers
 2905     // NOTE: cmpl is plenty here to provoke a segv
 2906     cmpptr(rax, Address(reg, 0));
 2907     // Note: should probably use testl(rax, Address(reg, 0));
 2908     //       may be shorter code (however, this version of
 2909     //       testl needs to be implemented first)
 2910   } else {
 2911     // nothing to do, (later) access of M[reg + offset]
 2912     // will provoke OS null exception if reg is null
 2913   }
 2914 }
 2915 
 2916 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
 2917   andptr(markword, markWord::inline_type_mask_in_place);
 2918   cmpptr(markword, markWord::inline_type_pattern);
 2919   jcc(Assembler::equal, is_inline_type);
 2920 }
 2921 
 2922 void MacroAssembler::test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type) {
 2923   movl(temp_reg, Address(klass, Klass::access_flags_offset()));
 2924   testl(temp_reg, JVM_ACC_IDENTITY);
 2925   jcc(Assembler::zero, is_inline_type);
 2926 }
 2927 
 2928 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type) {
 2929   testptr(object, object);
 2930   jcc(Assembler::zero, not_inline_type);
 2931   const int is_inline_type_mask = markWord::inline_type_pattern;
 2932   movptr(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
 2933   andptr(tmp, is_inline_type_mask);
 2934   cmpptr(tmp, is_inline_type_mask);
 2935   jcc(Assembler::notEqual, not_inline_type);
 2936 }
 2937 
 2938 void MacroAssembler::test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type) {
 2939 #ifdef ASSERT
 2940   {
 2941     Label done_check;
 2942     test_klass_is_inline_type(klass, temp_reg, done_check);
 2943     stop("test_klass_is_empty_inline_type with non inline type klass");
 2944     bind(done_check);
 2945   }
 2946 #endif
 2947   movl(temp_reg, Address(klass, InstanceKlass::misc_flags_offset()));
 2948   testl(temp_reg, InstanceKlassFlags::is_empty_inline_type_value());
 2949   jcc(Assembler::notZero, is_empty_inline_type);
 2950 }
 2951 
 2952 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
 2953   movl(temp_reg, flags);
 2954   testl(temp_reg, 1 << ResolvedFieldEntry::is_null_free_inline_type_shift);
 2955   jcc(Assembler::notEqual, is_null_free_inline_type);
 2956 }
 2957 
 2958 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
 2959   movl(temp_reg, flags);
 2960   testl(temp_reg, 1 << ResolvedFieldEntry::is_null_free_inline_type_shift);
 2961   jcc(Assembler::equal, not_null_free_inline_type);
 2962 }
 2963 
 2964 void MacroAssembler::test_field_is_flat(Register flags, Register temp_reg, Label& is_flat) {
 2965   movl(temp_reg, flags);
 2966   testl(temp_reg, 1 << ResolvedFieldEntry::is_flat_shift);
 2967   jcc(Assembler::notEqual, is_flat);
 2968 }
 2969 
 2970 void MacroAssembler::test_field_has_null_marker(Register flags, Register temp_reg, Label& has_null_marker) {
 2971   movl(temp_reg, flags);
 2972   testl(temp_reg, 1 << ResolvedFieldEntry::has_null_marker_shift);
 2973   jcc(Assembler::notEqual, has_null_marker);
 2974 }
 2975 
 2976 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label) {
 2977   Label test_mark_word;
 2978   // load mark word
 2979   movptr(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
 2980   // check displaced
 2981   testl(temp_reg, markWord::unlocked_value);
 2982   jccb(Assembler::notZero, test_mark_word);
 2983   // slow path use klass prototype
 2984   push(rscratch1);
 2985   load_prototype_header(temp_reg, oop, rscratch1);
 2986   pop(rscratch1);
 2987 
 2988   bind(test_mark_word);
 2989   testl(temp_reg, test_bit);
 2990   jcc((jmp_set) ? Assembler::notZero : Assembler::zero, jmp_label);
 2991 }
 2992 
 2993 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg,
 2994                                          Label& is_flat_array) {
 2995 #ifdef _LP64
 2996   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array);
 2997 #else
 2998   load_klass(temp_reg, oop, noreg);
 2999   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 3000   test_flat_array_layout(temp_reg, is_flat_array);
 3001 #endif
 3002 }
 3003 
 3004 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg,
 3005                                              Label& is_non_flat_array) {
 3006 #ifdef _LP64
 3007   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array);
 3008 #else
 3009   load_klass(temp_reg, oop, noreg);
 3010   movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
 3011   test_non_flat_array_layout(temp_reg, is_non_flat_array);
 3012 #endif
 3013 }
 3014 
 3015 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label&is_null_free_array) {
 3016 #ifdef _LP64
 3017   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
 3018 #else
 3019   Unimplemented();
 3020 #endif
 3021 }
 3022 
 3023 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
 3024 #ifdef _LP64
 3025   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
 3026 #else
 3027   Unimplemented();
 3028 #endif
 3029 }
 3030 
 3031 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) {
 3032   testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
 3033   jcc(Assembler::notZero, is_flat_array);
 3034 }
 3035 
 3036 void MacroAssembler::test_non_flat_array_layout(Register lh, Label& is_non_flat_array) {
 3037   testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
 3038   jcc(Assembler::zero, is_non_flat_array);
 3039 }
 3040 
 3041 void MacroAssembler::os_breakpoint() {
 3042   // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
 3043   // (e.g., MSVC can't call ps() otherwise)
 3044   call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
 3045 }
 3046 
 3047 void MacroAssembler::unimplemented(const char* what) {
 3048   const char* buf = nullptr;
 3049   {
 3050     ResourceMark rm;
 3051     stringStream ss;
 3052     ss.print("unimplemented: %s", what);
 3053     buf = code_string(ss.as_string());
 3054   }
 3055   stop(buf);
 3056 }
 3057 
 3058 #ifdef _LP64
 3059 #define XSTATE_BV 0x200
 3060 #endif

 4185 }
 4186 
 4187 // C++ bool manipulation
 4188 void MacroAssembler::testbool(Register dst) {
 4189   if(sizeof(bool) == 1)
 4190     testb(dst, 0xff);
 4191   else if(sizeof(bool) == 2) {
 4192     // testw implementation needed for two byte bools
 4193     ShouldNotReachHere();
 4194   } else if(sizeof(bool) == 4)
 4195     testl(dst, dst);
 4196   else
 4197     // unsupported
 4198     ShouldNotReachHere();
 4199 }
 4200 
 4201 void MacroAssembler::testptr(Register dst, Register src) {
 4202   LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
 4203 }
 4204 
 4205 // Object / value buffer allocation...
 4206 //
 4207 // Kills klass and rsi on LP64
 4208 void MacroAssembler::allocate_instance(Register klass, Register new_obj,
 4209                                        Register t1, Register t2,
 4210                                        bool clear_fields, Label& alloc_failed)
 4211 {
 4212   Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop;
 4213   Register layout_size = t1;
 4214   assert(new_obj == rax, "needs to be rax");
 4215   assert_different_registers(klass, new_obj, t1, t2);
 4216 
 4217   // get instance_size in InstanceKlass (scaled to a count of bytes)
 4218   movl(layout_size, Address(klass, Klass::layout_helper_offset()));
 4219   // test to see if it is malformed in some way
 4220   testl(layout_size, Klass::_lh_instance_slow_path_bit);
 4221   jcc(Assembler::notZero, slow_case_no_pop);
 4222 
 4223   // Allocate the instance:
 4224   //  If TLAB is enabled:
 4225   //    Try to allocate in the TLAB.
 4226   //    If fails, go to the slow path.
 4227   //  Else If inline contiguous allocations are enabled:
 4228   //    Try to allocate in eden.
 4229   //    If fails due to heap end, go to slow path.
 4230   //
 4231   //  If TLAB is enabled OR inline contiguous is enabled:
 4232   //    Initialize the allocation.
 4233   //    Exit.
 4234   //
 4235   //  Go to slow path.
 4236 
 4237   push(klass);
 4238   const Register thread = LP64_ONLY(r15_thread) NOT_LP64(klass);
 4239 #ifndef _LP64
 4240   if (UseTLAB) {
 4241     get_thread(thread);
 4242   }
 4243 #endif // _LP64
 4244 
 4245   if (UseTLAB) {
 4246     tlab_allocate(thread, new_obj, layout_size, 0, klass, t2, slow_case);
 4247     if (ZeroTLAB || (!clear_fields)) {
 4248       // the fields have been already cleared
 4249       jmp(initialize_header);
 4250     } else {
 4251       // initialize both the header and fields
 4252       jmp(initialize_object);
 4253     }
 4254   } else {
 4255     jmp(slow_case);
 4256   }
 4257 
 4258   // If UseTLAB is true, the object is created above and there is an initialize need.
 4259   // Otherwise, skip and go to the slow path.
 4260   if (UseTLAB) {
 4261     if (clear_fields) {
 4262       // The object is initialized before the header.  If the object size is
 4263       // zero, go directly to the header initialization.
 4264       bind(initialize_object);
 4265       decrement(layout_size, sizeof(oopDesc));
 4266       jcc(Assembler::zero, initialize_header);
 4267 
 4268       // Initialize topmost object field, divide size by 8, check if odd and
 4269       // test if zero.
 4270       Register zero = klass;
 4271       xorl(zero, zero);    // use zero reg to clear memory (shorter code)
 4272       shrl(layout_size, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd
 4273 
 4274   #ifdef ASSERT
 4275       // make sure instance_size was multiple of 8
 4276       Label L;
 4277       // Ignore partial flag stall after shrl() since it is debug VM
 4278       jcc(Assembler::carryClear, L);
 4279       stop("object size is not multiple of 2 - adjust this code");
 4280       bind(L);
 4281       // must be > 0, no extra check needed here
 4282   #endif
 4283 
 4284       // initialize remaining object fields: instance_size was a multiple of 8
 4285       {
 4286         Label loop;
 4287         bind(loop);
 4288         movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 1*oopSize), zero);
 4289         NOT_LP64(movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 2*oopSize), zero));
 4290         decrement(layout_size);
 4291         jcc(Assembler::notZero, loop);
 4292       }
 4293     } // clear_fields
 4294 
 4295     // initialize object header only.
 4296     bind(initialize_header);
 4297     pop(klass);
 4298     Register mark_word = t2;
 4299     movptr(mark_word, Address(klass, Klass::prototype_header_offset()));
 4300     movptr(Address(new_obj, oopDesc::mark_offset_in_bytes ()), mark_word);
 4301 #ifdef _LP64
 4302     xorl(rsi, rsi);                 // use zero reg to clear memory (shorter code)
 4303     store_klass_gap(new_obj, rsi);  // zero klass gap for compressed oops
 4304 #endif
 4305     movptr(t2, klass);         // preserve klass
 4306     store_klass(new_obj, t2, rscratch1);  // src klass reg is potentially compressed
 4307 
 4308     jmp(done);
 4309   }
 4310 
 4311   bind(slow_case);
 4312   pop(klass);
 4313   bind(slow_case_no_pop);
 4314   jmp(alloc_failed);
 4315 
 4316   bind(done);
 4317 }
 4318 
 4319 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
 4320 void MacroAssembler::tlab_allocate(Register thread, Register obj,
 4321                                    Register var_size_in_bytes,
 4322                                    int con_size_in_bytes,
 4323                                    Register t1,
 4324                                    Register t2,
 4325                                    Label& slow_case) {
 4326   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 4327   bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
 4328 }
 4329 
 4330 RegSet MacroAssembler::call_clobbered_gp_registers() {
 4331   RegSet regs;
 4332 #ifdef _LP64
 4333   regs += RegSet::of(rax, rcx, rdx);
 4334 #ifndef WINDOWS
 4335   regs += RegSet::of(rsi, rdi);
 4336 #endif
 4337   regs += RegSet::range(r8, r11);
 4338 #else

 4552     // clear topmost word (no jump would be needed if conditional assignment worked here)
 4553     movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
 4554     // index could be 0 now, must check again
 4555     jcc(Assembler::zero, done);
 4556     bind(even);
 4557   }
 4558 #endif // !_LP64
 4559   // initialize remaining object fields: index is a multiple of 2 now
 4560   {
 4561     Label loop;
 4562     bind(loop);
 4563     movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
 4564     NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
 4565     decrement(index);
 4566     jcc(Assembler::notZero, loop);
 4567   }
 4568 
 4569   bind(done);
 4570 }
 4571 
 4572 void MacroAssembler::get_inline_type_field_klass(Register klass, Register index, Register inline_klass) {
 4573   movptr(inline_klass, Address(klass, InstanceKlass::inline_type_field_klasses_offset()));
 4574 #ifdef ASSERT
 4575   {
 4576     Label done;
 4577     cmpptr(inline_klass, 0);
 4578     jcc(Assembler::notEqual, done);
 4579     stop("get_inline_type_field_klass contains no inline klass");
 4580     bind(done);
 4581   }
 4582 #endif
 4583   movptr(inline_klass, Address(inline_klass, index, Address::times_ptr, Array<InlineKlass*>::base_offset_in_bytes()));
 4584 }
 4585 
 4586 void MacroAssembler::get_default_value_oop(Register inline_klass, Register temp_reg, Register obj) {
 4587 #ifdef ASSERT
 4588   {
 4589     Label done_check;
 4590     test_klass_is_inline_type(inline_klass, temp_reg, done_check);
 4591     stop("get_default_value_oop from non inline type klass");
 4592     bind(done_check);
 4593   }
 4594 #endif
 4595   Register offset = temp_reg;
 4596   // Getting the offset of the pre-allocated default value
 4597   movptr(offset, Address(inline_klass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())));
 4598   movl(offset, Address(offset, in_bytes(InlineKlass::default_value_offset_offset())));
 4599 
 4600   // Getting the mirror
 4601   movptr(obj, Address(inline_klass, in_bytes(Klass::java_mirror_offset())));
 4602   resolve_oop_handle(obj, inline_klass);
 4603 
 4604   // Getting the pre-allocated default value from the mirror
 4605   Address field(obj, offset, Address::times_1);
 4606   load_heap_oop(obj, field);
 4607 }
 4608 
 4609 void MacroAssembler::get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj) {
 4610 #ifdef ASSERT
 4611   {
 4612     Label done_check;
 4613     test_klass_is_empty_inline_type(inline_klass, temp_reg, done_check);
 4614     stop("get_empty_value from non-empty inline klass");
 4615     bind(done_check);
 4616   }
 4617 #endif
 4618   get_default_value_oop(inline_klass, temp_reg, obj);
 4619 }
 4620 
 4621 
 4622 // Look up the method for a megamorphic invokeinterface call.
 4623 // The target method is determined by <intf_klass, itable_index>.
 4624 // The receiver klass is in recv_klass.
 4625 // On success, the result will be in method_result, and execution falls through.
 4626 // On failure, execution transfers to the given label.
 4627 void MacroAssembler::lookup_interface_method(Register recv_klass,
 4628                                              Register intf_klass,
 4629                                              RegisterOrConstant itable_index,
 4630                                              Register method_result,
 4631                                              Register scan_temp,
 4632                                              Label& L_no_such_interface,
 4633                                              bool return_method) {
 4634   assert_different_registers(recv_klass, intf_klass, scan_temp);
 4635   assert_different_registers(method_result, intf_klass, scan_temp);
 4636   assert(recv_klass != method_result || !return_method,
 4637          "recv_klass can be destroyed when method isn't needed");
 4638 
 4639   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 4640          "caller must use same register for non-constant itable index as for method");
 4641 

 5403   } else {
 5404     Label L;
 5405     jccb(negate_condition(cc), L);
 5406     movl(dst, src);
 5407     bind(L);
 5408   }
 5409 }
 5410 
 5411 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
 5412   if (VM_Version::supports_cmov()) {
 5413     cmovl(cc, dst, src);
 5414   } else {
 5415     Label L;
 5416     jccb(negate_condition(cc), L);
 5417     movl(dst, src);
 5418     bind(L);
 5419   }
 5420 }
 5421 
 5422 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 5423   if (!VerifyOops || VerifyAdapterSharing) {
 5424     // Below address of the code string confuses VerifyAdapterSharing
 5425     // because it may differ between otherwise equivalent adapters.
 5426     return;
 5427   }
 5428 
 5429   BLOCK_COMMENT("verify_oop {");
 5430 #ifdef _LP64
 5431   push(rscratch1);
 5432 #endif
 5433   push(rax);                          // save rax
 5434   push(reg);                          // pass register argument
 5435 
 5436   // Pass register number to verify_oop_subroutine
 5437   const char* b = nullptr;
 5438   {
 5439     ResourceMark rm;
 5440     stringStream ss;
 5441     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 5442     b = code_string(ss.as_string());
 5443   }
 5444   ExternalAddress buffer((address) b);
 5445   pushptr(buffer.addr(), rscratch1);
 5446 
 5447   // call indirectly to solve generation ordering problem

 5469   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 5470   int stackElementSize = Interpreter::stackElementSize;
 5471   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 5472 #ifdef ASSERT
 5473   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 5474   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 5475 #endif
 5476   Register             scale_reg    = noreg;
 5477   Address::ScaleFactor scale_factor = Address::no_scale;
 5478   if (arg_slot.is_constant()) {
 5479     offset += arg_slot.as_constant() * stackElementSize;
 5480   } else {
 5481     scale_reg    = arg_slot.as_register();
 5482     scale_factor = Address::times(stackElementSize);
 5483   }
 5484   offset += wordSize;           // return PC is on stack
 5485   return Address(rsp, scale_reg, scale_factor, offset);
 5486 }
 5487 
 5488 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 5489   if (!VerifyOops || VerifyAdapterSharing) {
 5490     // Below address of the code string confuses VerifyAdapterSharing
 5491     // because it may differ between otherwise equivalent adapters.
 5492     return;
 5493   }
 5494 
 5495 #ifdef _LP64
 5496   push(rscratch1);
 5497 #endif
 5498   push(rax); // save rax,
 5499   // addr may contain rsp so we will have to adjust it based on the push
 5500   // we just did (and on 64 bit we do two pushes)
 5501   // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
 5502   // stores rax into addr which is backwards of what was intended.
 5503   if (addr.uses(rsp)) {
 5504     lea(rax, addr);
 5505     pushptr(Address(rax, LP64_ONLY(2 *) BytesPerWord));
 5506   } else {
 5507     pushptr(addr);
 5508   }
 5509 
 5510   // Pass register number to verify_oop_subroutine
 5511   const char* b = nullptr;
 5512   {
 5513     ResourceMark rm;

 5960 
 5961 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
 5962   // get mirror
 5963   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
 5964   load_method_holder(mirror, method);
 5965   movptr(mirror, Address(mirror, mirror_offset));
 5966   resolve_oop_handle(mirror, tmp);
 5967 }
 5968 
 5969 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
 5970   load_method_holder(rresult, rmethod);
 5971   movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
 5972 }
 5973 
 5974 void MacroAssembler::load_method_holder(Register holder, Register method) {
 5975   movptr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
 5976   movptr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
 5977   movptr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
 5978 }
 5979 
 5980 void MacroAssembler::load_metadata(Register dst, Register src) {
 5981   if (UseCompressedClassPointers) {
 5982     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5983   } else {
 5984     movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5985   }
 5986 }
 5987 
 5988 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
 5989   assert_different_registers(src, tmp);
 5990   assert_different_registers(dst, tmp);
 5991 #ifdef _LP64
 5992   if (UseCompressedClassPointers) {
 5993     movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5994     decode_klass_not_null(dst, tmp);
 5995   } else
 5996 #endif
 5997   movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 5998 }
 5999 
 6000 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
 6001   load_klass(dst, src, tmp);
 6002   movptr(dst, Address(dst, Klass::prototype_header_offset()));
 6003 }
 6004 
 6005 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
 6006   assert_different_registers(src, tmp);
 6007   assert_different_registers(dst, tmp);
 6008 #ifdef _LP64
 6009   if (UseCompressedClassPointers) {
 6010     encode_klass_not_null(src, tmp);
 6011     movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 6012   } else
 6013 #endif
 6014     movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 6015 }
 6016 
 6017 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
 6018                                     Register tmp1, Register thread_tmp) {
 6019   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 6020   decorators = AccessInternal::decorator_fixup(decorators, type);
 6021   bool as_raw = (decorators & AS_RAW) != 0;
 6022   if (as_raw) {
 6023     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 6024   } else {
 6025     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
 6026   }
 6027 }
 6028 
 6029 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register val,
 6030                                      Register tmp1, Register tmp2, Register tmp3) {
 6031   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 6032   decorators = AccessInternal::decorator_fixup(decorators, type);
 6033   bool as_raw = (decorators & AS_RAW) != 0;
 6034   if (as_raw) {
 6035     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 6036   } else {
 6037     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
 6038   }
 6039 }
 6040 
 6041 void MacroAssembler::access_value_copy(DecoratorSet decorators, Register src, Register dst,
 6042                                        Register inline_klass) {
 6043   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 6044   bs->value_copy(this, decorators, src, dst, inline_klass);
 6045 }
 6046 
 6047 void MacroAssembler::first_field_offset(Register inline_klass, Register offset) {
 6048   movptr(offset, Address(inline_klass, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 6049   movl(offset, Address(offset, InlineKlass::first_field_offset_offset()));
 6050 }
 6051 
 6052 void MacroAssembler::data_for_oop(Register oop, Register data, Register inline_klass) {
 6053   // ((address) (void*) o) + vk->first_field_offset();
 6054   Register offset = (data == oop) ? rscratch1 : data;
 6055   first_field_offset(inline_klass, offset);
 6056   if (data == oop) {
 6057     addptr(data, offset);
 6058   } else {
 6059     lea(data, Address(oop, offset));
 6060   }
 6061 }
 6062 
 6063 void MacroAssembler::data_for_value_array_index(Register array, Register array_klass,
 6064                                                 Register index, Register data) {
 6065   assert(index != rcx, "index needs to shift by rcx");
 6066   assert_different_registers(array, array_klass, index);
 6067   assert_different_registers(rcx, array, index);
 6068 
 6069   // array->base() + (index << Klass::layout_helper_log2_element_size(lh));
 6070   movl(rcx, Address(array_klass, Klass::layout_helper_offset()));
 6071 
 6072   // Klass::layout_helper_log2_element_size(lh)
 6073   // (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask;
 6074   shrl(rcx, Klass::_lh_log2_element_size_shift);
 6075   andl(rcx, Klass::_lh_log2_element_size_mask);
 6076   shlptr(index); // index << rcx
 6077 
 6078   lea(data, Address(array, index, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_PRIMITIVE_OBJECT)));
 6079 }
 6080 
 6081 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
 6082                                    Register thread_tmp, DecoratorSet decorators) {
 6083   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
 6084 }
 6085 
 6086 // Doesn't do verification, generates fixed size code
 6087 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
 6088                                             Register thread_tmp, DecoratorSet decorators) {
 6089   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
 6090 }
 6091 
 6092 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
 6093                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
 6094   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
 6095 }
 6096 
 6097 // Used for storing nulls.
 6098 void MacroAssembler::store_heap_oop_null(Address dst) {
 6099   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
 6100 }

 6400 
 6401 void MacroAssembler::reinit_heapbase() {
 6402   if (UseCompressedOops) {
 6403     if (Universe::heap() != nullptr) {
 6404       if (CompressedOops::base() == nullptr) {
 6405         MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
 6406       } else {
 6407         mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
 6408       }
 6409     } else {
 6410       movptr(r12_heapbase, ExternalAddress(CompressedOops::ptrs_base_addr()));
 6411     }
 6412   }
 6413 }
 6414 
 6415 #endif // _LP64
 6416 
 6417 #if COMPILER2_OR_JVMCI
 6418 
 6419 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
 6420 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask) {
 6421   // cnt - number of qwords (8-byte words).
 6422   // base - start address, qword aligned.
 6423   Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
 6424   bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
 6425   if (use64byteVector) {
 6426     evpbroadcastq(xtmp, val, AVX_512bit);
 6427   } else if (MaxVectorSize >= 32) {
 6428     movdq(xtmp, val);
 6429     punpcklqdq(xtmp, xtmp);
 6430     vinserti128_high(xtmp, xtmp);
 6431   } else {
 6432     movdq(xtmp, val);
 6433     punpcklqdq(xtmp, xtmp);
 6434   }
 6435   jmp(L_zero_64_bytes);
 6436 
 6437   BIND(L_loop);
 6438   if (MaxVectorSize >= 32) {
 6439     fill64(base, 0, xtmp, use64byteVector);
 6440   } else {
 6441     movdqu(Address(base,  0), xtmp);
 6442     movdqu(Address(base, 16), xtmp);
 6443     movdqu(Address(base, 32), xtmp);
 6444     movdqu(Address(base, 48), xtmp);
 6445   }
 6446   addptr(base, 64);
 6447 
 6448   BIND(L_zero_64_bytes);
 6449   subptr(cnt, 8);
 6450   jccb(Assembler::greaterEqual, L_loop);
 6451 
 6452   // Copy trailing 64 bytes
 6453   if (use64byteVector) {
 6454     addptr(cnt, 8);
 6455     jccb(Assembler::equal, L_end);
 6456     fill64_masked(3, base, 0, xtmp, mask, cnt, val, true);
 6457     jmp(L_end);
 6458   } else {
 6459     addptr(cnt, 4);
 6460     jccb(Assembler::less, L_tail);
 6461     if (MaxVectorSize >= 32) {
 6462       vmovdqu(Address(base, 0), xtmp);
 6463     } else {
 6464       movdqu(Address(base,  0), xtmp);
 6465       movdqu(Address(base, 16), xtmp);
 6466     }
 6467   }
 6468   addptr(base, 32);
 6469   subptr(cnt, 4);
 6470 
 6471   BIND(L_tail);
 6472   addptr(cnt, 4);
 6473   jccb(Assembler::lessEqual, L_end);
 6474   if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
 6475     fill32_masked(3, base, 0, xtmp, mask, cnt, val);
 6476   } else {
 6477     decrement(cnt);
 6478 
 6479     BIND(L_sloop);
 6480     movq(Address(base, 0), xtmp);
 6481     addptr(base, 8);
 6482     decrement(cnt);
 6483     jccb(Assembler::greaterEqual, L_sloop);
 6484   }
 6485   BIND(L_end);
 6486 }
 6487 
 6488 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
 6489   assert(InlineTypeReturnedAsFields, "Inline types should never be returned as fields");
 6490   // An inline type might be returned. If fields are in registers we
 6491   // need to allocate an inline type instance and initialize it with
 6492   // the value of the fields.
 6493   Label skip;
 6494   // We only need a new buffered inline type if a new one is not returned
 6495   testptr(rax, 1);
 6496   jcc(Assembler::zero, skip);
 6497   int call_offset = -1;
 6498 
 6499 #ifdef _LP64
 6500   // The following code is similar to allocate_instance but has some slight differences,
 6501   // e.g. object size is always not zero, sometimes it's constant; storing klass ptr after
 6502   // allocating is not necessary if vk != nullptr, etc. allocate_instance is not aware of these.
 6503   Label slow_case;
 6504   // 1. Try to allocate a new buffered inline instance either from TLAB or eden space
 6505   mov(rscratch1, rax); // save rax for slow_case since *_allocate may corrupt it when allocation failed
 6506   if (vk != nullptr) {
 6507     // Called from C1, where the return type is statically known.
 6508     movptr(rbx, (intptr_t)vk->get_InlineKlass());
 6509     jint lh = vk->layout_helper();
 6510     assert(lh != Klass::_lh_neutral_value, "inline class in return type must have been resolved");
 6511     if (UseTLAB && !Klass::layout_helper_needs_slow_path(lh)) {
 6512       tlab_allocate(r15_thread, rax, noreg, lh, r13, r14, slow_case);
 6513     } else {
 6514       jmp(slow_case);
 6515     }
 6516   } else {
 6517     // Call from interpreter. RAX contains ((the InlineKlass* of the return type) | 0x01)
 6518     mov(rbx, rax);
 6519     andptr(rbx, -2);
 6520     if (UseTLAB) {
 6521       movl(r14, Address(rbx, Klass::layout_helper_offset()));
 6522       testl(r14, Klass::_lh_instance_slow_path_bit);
 6523       jcc(Assembler::notZero, slow_case);
 6524       tlab_allocate(r15_thread, rax, r14, 0, r13, r14, slow_case);
 6525     } else {
 6526       jmp(slow_case);
 6527     }
 6528   }
 6529   if (UseTLAB) {
 6530     // 2. Initialize buffered inline instance header
 6531     Register buffer_obj = rax;
 6532     movptr(Address(buffer_obj, oopDesc::mark_offset_in_bytes()), (intptr_t)markWord::inline_type_prototype().value());
 6533     xorl(r13, r13);
 6534     store_klass_gap(buffer_obj, r13);
 6535     if (vk == nullptr) {
 6536       // store_klass corrupts rbx(klass), so save it in r13 for later use (interpreter case only).
 6537       mov(r13, rbx);
 6538     }
 6539     store_klass(buffer_obj, rbx, rscratch1);
 6540     // 3. Initialize its fields with an inline class specific handler
 6541     if (vk != nullptr) {
 6542       call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint.
 6543     } else {
 6544       movptr(rbx, Address(r13, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 6545       movptr(rbx, Address(rbx, InlineKlass::pack_handler_offset()));
 6546       call(rbx);
 6547     }
 6548     jmp(skip);
 6549   }
 6550   bind(slow_case);
 6551   // We failed to allocate a new inline type, fall back to a runtime
 6552   // call. Some oop field may be live in some registers but we can't
 6553   // tell. That runtime call will take care of preserving them
 6554   // across a GC if there's one.
 6555   mov(rax, rscratch1);
 6556 #endif
 6557 
 6558   if (from_interpreter) {
 6559     super_call_VM_leaf(StubRoutines::store_inline_type_fields_to_buf());
 6560   } else {
 6561     call(RuntimeAddress(StubRoutines::store_inline_type_fields_to_buf()));
 6562     call_offset = offset();
 6563   }
 6564 
 6565   bind(skip);
 6566   return call_offset;
 6567 }
 6568 
 6569 // Move a value between registers/stack slots and update the reg_state
 6570 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
 6571   assert(from->is_valid() && to->is_valid(), "source and destination must be valid");
 6572   if (reg_state[to->value()] == reg_written) {
 6573     return true; // Already written
 6574   }
 6575   if (from != to && bt != T_VOID) {
 6576     if (reg_state[to->value()] == reg_readonly) {
 6577       return false; // Not yet writable
 6578     }
 6579     if (from->is_reg()) {
 6580       if (to->is_reg()) {
 6581         if (from->is_XMMRegister()) {
 6582           if (bt == T_DOUBLE) {
 6583             movdbl(to->as_XMMRegister(), from->as_XMMRegister());
 6584           } else {
 6585             assert(bt == T_FLOAT, "must be float");
 6586             movflt(to->as_XMMRegister(), from->as_XMMRegister());
 6587           }
 6588         } else {
 6589           movq(to->as_Register(), from->as_Register());
 6590         }
 6591       } else {
 6592         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6593         Address to_addr = Address(rsp, st_off);
 6594         if (from->is_XMMRegister()) {
 6595           if (bt == T_DOUBLE) {
 6596             movdbl(to_addr, from->as_XMMRegister());
 6597           } else {
 6598             assert(bt == T_FLOAT, "must be float");
 6599             movflt(to_addr, from->as_XMMRegister());
 6600           }
 6601         } else {
 6602           movq(to_addr, from->as_Register());
 6603         }
 6604       }
 6605     } else {
 6606       Address from_addr = Address(rsp, from->reg2stack() * VMRegImpl::stack_slot_size + wordSize);
 6607       if (to->is_reg()) {
 6608         if (to->is_XMMRegister()) {
 6609           if (bt == T_DOUBLE) {
 6610             movdbl(to->as_XMMRegister(), from_addr);
 6611           } else {
 6612             assert(bt == T_FLOAT, "must be float");
 6613             movflt(to->as_XMMRegister(), from_addr);
 6614           }
 6615         } else {
 6616           movq(to->as_Register(), from_addr);
 6617         }
 6618       } else {
 6619         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6620         movq(r13, from_addr);
 6621         movq(Address(rsp, st_off), r13);
 6622       }
 6623     }
 6624   }
 6625   // Update register states
 6626   reg_state[from->value()] = reg_writable;
 6627   reg_state[to->value()] = reg_written;
 6628   return true;
 6629 }
 6630 
 6631 // Calculate the extra stack space required for packing or unpacking inline
 6632 // args and adjust the stack pointer
 6633 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
 6634   // Two additional slots to account for return address
 6635   int sp_inc = (args_on_stack + 2) * VMRegImpl::stack_slot_size;
 6636   sp_inc = align_up(sp_inc, StackAlignmentInBytes);
 6637   // Save the return address, adjust the stack (make sure it is properly
 6638   // 16-byte aligned) and copy the return address to the new top of the stack.
 6639   // The stack will be repaired on return (see MacroAssembler::remove_frame).
 6640   assert(sp_inc > 0, "sanity");
 6641   pop(r13);
 6642   subptr(rsp, sp_inc);
 6643   push(r13);
 6644   return sp_inc;
 6645 }
 6646 
 6647 // Read all fields from an inline type buffer and store the field values in registers/stack slots.
 6648 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
 6649                                           VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
 6650                                           RegState reg_state[]) {
 6651   assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter");
 6652   assert(from->is_valid(), "source must be valid");
 6653   bool progress = false;
 6654 #ifdef ASSERT
 6655   const int start_offset = offset();
 6656 #endif
 6657 
 6658   Label L_null, L_notNull;
 6659   // Don't use r14 as tmp because it's used for spilling (see MacroAssembler::spill_reg_for)
 6660   Register tmp1 = r10;
 6661   Register tmp2 = r13;
 6662   Register fromReg = noreg;
 6663   ScalarizedInlineArgsStream stream(sig, sig_index, to, to_count, to_index, -1);
 6664   bool done = true;
 6665   bool mark_done = true;
 6666   VMReg toReg;
 6667   BasicType bt;
 6668   // Check if argument requires a null check
 6669   bool null_check = false;
 6670   VMReg nullCheckReg;
 6671   while (stream.next(nullCheckReg, bt)) {
 6672     if (sig->at(stream.sig_index())._offset == -1) {
 6673       null_check = true;
 6674       break;
 6675     }
 6676   }
 6677   stream.reset(sig_index, to_index);
 6678   while (stream.next(toReg, bt)) {
 6679     assert(toReg->is_valid(), "destination must be valid");
 6680     int idx = (int)toReg->value();
 6681     if (reg_state[idx] == reg_readonly) {
 6682       if (idx != from->value()) {
 6683         mark_done = false;
 6684       }
 6685       done = false;
 6686       continue;
 6687     } else if (reg_state[idx] == reg_written) {
 6688       continue;
 6689     }
 6690     assert(reg_state[idx] == reg_writable, "must be writable");
 6691     reg_state[idx] = reg_written;
 6692     progress = true;
 6693 
 6694     if (fromReg == noreg) {
 6695       if (from->is_reg()) {
 6696         fromReg = from->as_Register();
 6697       } else {
 6698         int st_off = from->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6699         movq(tmp1, Address(rsp, st_off));
 6700         fromReg = tmp1;
 6701       }
 6702       if (null_check) {
 6703         // Nullable inline type argument, emit null check
 6704         testptr(fromReg, fromReg);
 6705         jcc(Assembler::zero, L_null);
 6706       }
 6707     }
 6708     int off = sig->at(stream.sig_index())._offset;
 6709     if (off == -1) {
 6710       assert(null_check, "Missing null check at");
 6711       if (toReg->is_stack()) {
 6712         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6713         movq(Address(rsp, st_off), 1);
 6714       } else {
 6715         movq(toReg->as_Register(), 1);
 6716       }
 6717       continue;
 6718     }
 6719     assert(off > 0, "offset in object should be positive");
 6720     Address fromAddr = Address(fromReg, off);
 6721     if (!toReg->is_XMMRegister()) {
 6722       Register dst = toReg->is_stack() ? tmp2 : toReg->as_Register();
 6723       if (is_reference_type(bt)) {
 6724         load_heap_oop(dst, fromAddr);
 6725       } else {
 6726         bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN);
 6727         load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed);
 6728       }
 6729       if (toReg->is_stack()) {
 6730         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6731         movq(Address(rsp, st_off), dst);
 6732       }
 6733     } else if (bt == T_DOUBLE) {
 6734       movdbl(toReg->as_XMMRegister(), fromAddr);
 6735     } else {
 6736       assert(bt == T_FLOAT, "must be float");
 6737       movflt(toReg->as_XMMRegister(), fromAddr);
 6738     }
 6739   }
 6740   if (progress && null_check) {
 6741     if (done) {
 6742       jmp(L_notNull);
 6743       bind(L_null);
 6744       // Set IsInit field to zero to signal that the argument is null.
 6745       // Also set all oop fields to zero to make the GC happy.
 6746       stream.reset(sig_index, to_index);
 6747       while (stream.next(toReg, bt)) {
 6748         if (sig->at(stream.sig_index())._offset == -1 ||
 6749             bt == T_OBJECT || bt == T_ARRAY) {
 6750           if (toReg->is_stack()) {
 6751             int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6752             movq(Address(rsp, st_off), 0);
 6753           } else {
 6754             xorq(toReg->as_Register(), toReg->as_Register());
 6755           }
 6756         }
 6757       }
 6758       bind(L_notNull);
 6759     } else {
 6760       bind(L_null);
 6761     }
 6762   }
 6763 
 6764   sig_index = stream.sig_index();
 6765   to_index = stream.regs_index();
 6766 
 6767   if (mark_done && reg_state[from->value()] != reg_written) {
 6768     // This is okay because no one else will write to that slot
 6769     reg_state[from->value()] = reg_writable;
 6770   }
 6771   from_index--;
 6772   assert(progress || (start_offset == offset()), "should not emit code");
 6773   return done;
 6774 }
 6775 
 6776 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
 6777                                         VMRegPair* from, int from_count, int& from_index, VMReg to,
 6778                                         RegState reg_state[], Register val_array) {
 6779   assert(sig->at(sig_index)._bt == T_METADATA, "should be at delimiter");
 6780   assert(to->is_valid(), "destination must be valid");
 6781 
 6782   if (reg_state[to->value()] == reg_written) {
 6783     skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
 6784     return true; // Already written
 6785   }
 6786 
 6787   // TODO 8284443 Isn't it an issue if below code uses r14 as tmp when it contains a spilled value?
 6788   // Be careful with r14 because it's used for spilling (see MacroAssembler::spill_reg_for).
 6789   Register val_obj_tmp = r11;
 6790   Register from_reg_tmp = r14;
 6791   Register tmp1 = r10;
 6792   Register tmp2 = r13;
 6793   Register tmp3 = rbx;
 6794   Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register();
 6795 
 6796   assert_different_registers(val_obj_tmp, from_reg_tmp, tmp1, tmp2, tmp3, val_array);
 6797 
 6798   if (reg_state[to->value()] == reg_readonly) {
 6799     if (!is_reg_in_unpacked_fields(sig, sig_index, to, from, from_count, from_index)) {
 6800       skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
 6801       return false; // Not yet writable
 6802     }
 6803     val_obj = val_obj_tmp;
 6804   }
 6805 
 6806   int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_OBJECT);
 6807   load_heap_oop(val_obj, Address(val_array, index));
 6808 
 6809   ScalarizedInlineArgsStream stream(sig, sig_index, from, from_count, from_index);
 6810   VMReg fromReg;
 6811   BasicType bt;
 6812   Label L_null;
 6813   while (stream.next(fromReg, bt)) {
 6814     assert(fromReg->is_valid(), "source must be valid");
 6815     reg_state[fromReg->value()] = reg_writable;
 6816 
 6817     int off = sig->at(stream.sig_index())._offset;
 6818     if (off == -1) {
 6819       // Nullable inline type argument, emit null check
 6820       Label L_notNull;
 6821       if (fromReg->is_stack()) {
 6822         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6823         testb(Address(rsp, ld_off), 1);
 6824       } else {
 6825         testb(fromReg->as_Register(), 1);
 6826       }
 6827       jcc(Assembler::notZero, L_notNull);
 6828       movptr(val_obj, 0);
 6829       jmp(L_null);
 6830       bind(L_notNull);
 6831       continue;
 6832     }
 6833 
 6834     assert(off > 0, "offset in object should be positive");
 6835     size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
 6836 
 6837     Address dst(val_obj, off);
 6838     if (!fromReg->is_XMMRegister()) {
 6839       Register src;
 6840       if (fromReg->is_stack()) {
 6841         src = from_reg_tmp;
 6842         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
 6843         load_sized_value(src, Address(rsp, ld_off), size_in_bytes, /* is_signed */ false);
 6844       } else {
 6845         src = fromReg->as_Register();
 6846       }
 6847       assert_different_registers(dst.base(), src, tmp1, tmp2, tmp3, val_array);
 6848       if (is_reference_type(bt)) {
 6849         store_heap_oop(dst, src, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
 6850       } else {
 6851         store_sized_value(dst, src, size_in_bytes);
 6852       }
 6853     } else if (bt == T_DOUBLE) {
 6854       movdbl(dst, fromReg->as_XMMRegister());
 6855     } else {
 6856       assert(bt == T_FLOAT, "must be float");
 6857       movflt(dst, fromReg->as_XMMRegister());
 6858     }
 6859   }
 6860   bind(L_null);
 6861   sig_index = stream.sig_index();
 6862   from_index = stream.regs_index();
 6863 
 6864   assert(reg_state[to->value()] == reg_writable, "must have already been read");
 6865   bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state);
 6866   assert(success, "to register must be writeable");
 6867   return true;
 6868 }
 6869 
 6870 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
 6871   return reg->is_XMMRegister() ? xmm8->as_VMReg() : r14->as_VMReg();
 6872 }
 6873 
 6874 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
 6875   assert((initial_framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
 6876   if (needs_stack_repair) {
 6877     movq(rbp, Address(rsp, initial_framesize));
 6878     // The stack increment resides just below the saved rbp
 6879     addq(rsp, Address(rsp, initial_framesize - wordSize));
 6880   } else {
 6881     if (initial_framesize > 0) {
 6882       addq(rsp, initial_framesize);
 6883     }
 6884     pop(rbp);
 6885   }
 6886 }
 6887 
 6888 // Clearing constant sized memory using YMM/ZMM registers.
 6889 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
 6890   assert(UseAVX > 2 && VM_Version::supports_avx512vl(), "");
 6891   bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
 6892 
 6893   int vector64_count = (cnt & (~0x7)) >> 3;
 6894   cnt = cnt & 0x7;
 6895   const int fill64_per_loop = 4;
 6896   const int max_unrolled_fill64 = 8;
 6897 
 6898   // 64 byte initialization loop.
 6899   vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
 6900   int start64 = 0;
 6901   if (vector64_count > max_unrolled_fill64) {
 6902     Label LOOP;
 6903     Register index = rtmp;
 6904 
 6905     start64 = vector64_count - (vector64_count % fill64_per_loop);
 6906 
 6907     movl(index, 0);

 6957         break;
 6958       case 7:
 6959         if (use64byteVector) {
 6960           movl(rtmp, 0x7F);
 6961           kmovwl(mask, rtmp);
 6962           evmovdqu(T_LONG, mask, Address(base, disp), xtmp, true, Assembler::AVX_512bit);
 6963         } else {
 6964           evmovdqu(T_LONG, k0, Address(base, disp), xtmp, false, Assembler::AVX_256bit);
 6965           movl(rtmp, 0x7);
 6966           kmovwl(mask, rtmp);
 6967           evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, true, Assembler::AVX_256bit);
 6968         }
 6969         break;
 6970       default:
 6971         fatal("Unexpected length : %d\n",cnt);
 6972         break;
 6973     }
 6974   }
 6975 }
 6976 
 6977 void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp,
 6978                                bool is_large, bool word_copy_only, KRegister mask) {
 6979   // cnt      - number of qwords (8-byte words).
 6980   // base     - start address, qword aligned.
 6981   // is_large - if optimizers know cnt is larger than InitArrayShortSize
 6982   assert(base==rdi, "base register must be edi for rep stos");
 6983   assert(val==rax,   "val register must be eax for rep stos");
 6984   assert(cnt==rcx,   "cnt register must be ecx for rep stos");
 6985   assert(InitArrayShortSize % BytesPerLong == 0,
 6986     "InitArrayShortSize should be the multiple of BytesPerLong");
 6987 
 6988   Label DONE;



 6989 
 6990   if (!is_large) {
 6991     Label LOOP, LONG;
 6992     cmpptr(cnt, InitArrayShortSize/BytesPerLong);
 6993     jccb(Assembler::greater, LONG);
 6994 
 6995     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 6996 
 6997     decrement(cnt);
 6998     jccb(Assembler::negative, DONE); // Zero length
 6999 
 7000     // Use individual pointer-sized stores for small counts:
 7001     BIND(LOOP);
 7002     movptr(Address(base, cnt, Address::times_ptr), val);
 7003     decrement(cnt);
 7004     jccb(Assembler::greaterEqual, LOOP);
 7005     jmpb(DONE);
 7006 
 7007     BIND(LONG);
 7008   }
 7009 
 7010   // Use longer rep-prefixed ops for non-small counts:
 7011   if (UseFastStosb && !word_copy_only) {
 7012     shlptr(cnt, 3); // convert to number of bytes
 7013     rep_stosb();
 7014   } else if (UseXMMForObjInit) {
 7015     xmm_clear_mem(base, cnt, val, xtmp, mask);
 7016   } else {
 7017     NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
 7018     rep_stos();
 7019   }
 7020 
 7021   BIND(DONE);
 7022 }
 7023 
 7024 #endif //COMPILER2_OR_JVMCI
 7025 
 7026 
 7027 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 7028                                    Register to, Register value, Register count,
 7029                                    Register rtmp, XMMRegister xtmp) {
 7030   ShortBranchVerifier sbv(this);
 7031   assert_different_registers(to, value, count, rtmp);
 7032   Label L_exit;
 7033   Label L_fill_2_bytes, L_fill_4_bytes;
 7034 
 7035 #if defined(COMPILER2) && defined(_LP64)

11045 
11046   // Load top.
11047   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
11048 
11049   // Check if the lock-stack is full.
11050   cmpl(top, LockStack::end_offset());
11051   jcc(Assembler::greaterEqual, slow);
11052 
11053   // Check for recursion.
11054   cmpptr(obj, Address(thread, top, Address::times_1, -oopSize));
11055   jcc(Assembler::equal, push);
11056 
11057   // Check header for monitor (0b10).
11058   testptr(reg_rax, markWord::monitor_value);
11059   jcc(Assembler::notZero, slow);
11060 
11061   // Try to lock. Transition lock bits 0b01 => 0b00
11062   movptr(tmp, reg_rax);
11063   andptr(tmp, ~(int32_t)markWord::unlocked_value);
11064   orptr(reg_rax, markWord::unlocked_value);
11065   if (EnableValhalla) {
11066     // Mask inline_type bit such that we go to the slow path if object is an inline type
11067     andptr(reg_rax, ~((int) markWord::inline_type_bit_in_place));
11068   }
11069   lock(); cmpxchgptr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
11070   jcc(Assembler::notEqual, slow);
11071 
11072   // Restore top, CAS clobbers register.
11073   movl(top, Address(thread, JavaThread::lock_stack_top_offset()));
11074 
11075   bind(push);
11076   // After successful lock, push object on lock-stack.
11077   movptr(Address(thread, top), obj);
11078   incrementl(top, oopSize);
11079   movl(Address(thread, JavaThread::lock_stack_top_offset()), top);
11080 }
11081 
11082 // Implements lightweight-unlocking.
11083 //
11084 // obj: the object to be unlocked
11085 // reg_rax: rax
11086 // thread: the thread
11087 // tmp: a temporary register
11088 //
< prev index next >