12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "jvm.h"
27 #include "asm/assembler.hpp"
28 #include "asm/assembler.inline.hpp"
29 #include "c1/c1_FrameMap.hpp"
30 #include "compiler/compiler_globals.hpp"
31 #include "compiler/disassembler.hpp"
32 #include "gc/shared/barrierSet.hpp"
33 #include "gc/shared/barrierSetAssembler.hpp"
34 #include "gc/shared/collectedHeap.inline.hpp"
35 #include "gc/shared/tlab_globals.hpp"
36 #include "interpreter/bytecodeHistogram.hpp"
37 #include "interpreter/interpreter.hpp"
38 #include "memory/resourceArea.hpp"
39 #include "memory/universe.hpp"
40 #include "oops/accessDecorators.hpp"
41 #include "oops/compressedOops.inline.hpp"
42 #include "oops/klass.inline.hpp"
43 #include "prims/methodHandles.hpp"
44 #include "runtime/flags/flagSetting.hpp"
45 #include "runtime/interfaceSupport.inline.hpp"
46 #include "runtime/jniHandles.hpp"
47 #include "runtime/objectMonitor.hpp"
48 #include "runtime/os.hpp"
49 #include "runtime/safepoint.hpp"
50 #include "runtime/safepointMechanism.hpp"
51 #include "runtime/sharedRuntime.hpp"
52 #include "runtime/stubRoutines.hpp"
53 #include "runtime/thread.hpp"
54 #include "utilities/macros.hpp"
55 #include "crc32c.h"
56
57 #ifdef PRODUCT
58 #define BLOCK_COMMENT(str) /* nothing */
59 #define STOP(error) stop(error)
60 #else
61 #define BLOCK_COMMENT(str) block_comment(str)
62 #define STOP(error) block_comment(error); stop(error)
63 #endif
64
65 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
66
67 #ifdef ASSERT
68 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
69 #endif
70
71 static Assembler::Condition reverse[] = {
72 Assembler::noOverflow /* overflow = 0x0 */ ,
73 Assembler::overflow /* noOverflow = 0x1 */ ,
74 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ ,
75 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ ,
1619 }
1620
1621 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1622
1623 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1624 pass_arg1(this, arg_1);
1625 pass_arg0(this, arg_0);
1626 call_VM_leaf(entry_point, 2);
1627 }
1628
1629 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1630 LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
1631 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1632 pass_arg2(this, arg_2);
1633 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1634 pass_arg1(this, arg_1);
1635 pass_arg0(this, arg_0);
1636 call_VM_leaf(entry_point, 3);
1637 }
1638
1639 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1640 pass_arg0(this, arg_0);
1641 MacroAssembler::call_VM_leaf_base(entry_point, 1);
1642 }
1643
1644 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1645
1646 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1647 pass_arg1(this, arg_1);
1648 pass_arg0(this, arg_0);
1649 MacroAssembler::call_VM_leaf_base(entry_point, 2);
1650 }
1651
1652 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1653 LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
1654 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1655 pass_arg2(this, arg_2);
1656 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1657 pass_arg1(this, arg_1);
1658 pass_arg0(this, arg_0);
2713 lea(rscratch1, src);
2714 Assembler::mulss(dst, Address(rscratch1, 0));
2715 }
2716 }
2717
2718 void MacroAssembler::null_check(Register reg, int offset) {
2719 if (needs_explicit_null_check(offset)) {
2720 // provoke OS NULL exception if reg = NULL by
2721 // accessing M[reg] w/o changing any (non-CC) registers
2722 // NOTE: cmpl is plenty here to provoke a segv
2723 cmpptr(rax, Address(reg, 0));
2724 // Note: should probably use testl(rax, Address(reg, 0));
2725 // may be shorter code (however, this version of
2726 // testl needs to be implemented first)
2727 } else {
2728 // nothing to do, (later) access of M[reg + offset]
2729 // will provoke OS NULL exception if reg = NULL
2730 }
2731 }
2732
2733 void MacroAssembler::os_breakpoint() {
2734 // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
2735 // (e.g., MSVC can't call ps() otherwise)
2736 call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
2737 }
2738
2739 void MacroAssembler::unimplemented(const char* what) {
2740 const char* buf = NULL;
2741 {
2742 ResourceMark rm;
2743 stringStream ss;
2744 ss.print("unimplemented: %s", what);
2745 buf = code_string(ss.as_string());
2746 }
2747 stop(buf);
2748 }
2749
2750 #ifdef _LP64
2751 #define XSTATE_BV 0x200
2752 #endif
3551 }
3552
3553 // C++ bool manipulation
3554 void MacroAssembler::testbool(Register dst) {
3555 if(sizeof(bool) == 1)
3556 testb(dst, 0xff);
3557 else if(sizeof(bool) == 2) {
3558 // testw implementation needed for two byte bools
3559 ShouldNotReachHere();
3560 } else if(sizeof(bool) == 4)
3561 testl(dst, dst);
3562 else
3563 // unsupported
3564 ShouldNotReachHere();
3565 }
3566
3567 void MacroAssembler::testptr(Register dst, Register src) {
3568 LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
3569 }
3570
3571 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
3572 void MacroAssembler::tlab_allocate(Register thread, Register obj,
3573 Register var_size_in_bytes,
3574 int con_size_in_bytes,
3575 Register t1,
3576 Register t2,
3577 Label& slow_case) {
3578 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3579 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3580 }
3581
3582 RegSet MacroAssembler::call_clobbered_gp_registers() {
3583 RegSet regs;
3584 #ifdef _LP64
3585 regs += RegSet::of(rax, rcx, rdx);
3586 #ifndef WINDOWS
3587 regs += RegSet::of(rsi, rdi);
3588 #endif
3589 regs += RegSet::range(r8, r11);
3590 #else
3812 // clear topmost word (no jump would be needed if conditional assignment worked here)
3813 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
3814 // index could be 0 now, must check again
3815 jcc(Assembler::zero, done);
3816 bind(even);
3817 }
3818 #endif // !_LP64
3819 // initialize remaining object fields: index is a multiple of 2 now
3820 {
3821 Label loop;
3822 bind(loop);
3823 movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
3824 NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
3825 decrement(index);
3826 jcc(Assembler::notZero, loop);
3827 }
3828
3829 bind(done);
3830 }
3831
3832 // Look up the method for a megamorphic invokeinterface call.
3833 // The target method is determined by <intf_klass, itable_index>.
3834 // The receiver klass is in recv_klass.
3835 // On success, the result will be in method_result, and execution falls through.
3836 // On failure, execution transfers to the given label.
3837 void MacroAssembler::lookup_interface_method(Register recv_klass,
3838 Register intf_klass,
3839 RegisterOrConstant itable_index,
3840 Register method_result,
3841 Register scan_temp,
3842 Label& L_no_such_interface,
3843 bool return_method) {
3844 assert_different_registers(recv_klass, intf_klass, scan_temp);
3845 assert_different_registers(method_result, intf_klass, scan_temp);
3846 assert(recv_klass != method_result || !return_method,
3847 "recv_klass can be destroyed when method isn't needed");
3848
3849 assert(itable_index.is_constant() || itable_index.as_register() == method_result,
3850 "caller must use same register for non-constant itable index as for method");
3851
4160 } else {
4161 Label L;
4162 jccb(negate_condition(cc), L);
4163 movl(dst, src);
4164 bind(L);
4165 }
4166 }
4167
4168 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
4169 if (VM_Version::supports_cmov()) {
4170 cmovl(cc, dst, src);
4171 } else {
4172 Label L;
4173 jccb(negate_condition(cc), L);
4174 movl(dst, src);
4175 bind(L);
4176 }
4177 }
4178
4179 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
4180 if (!VerifyOops) return;
4181
4182 // Pass register number to verify_oop_subroutine
4183 const char* b = NULL;
4184 {
4185 ResourceMark rm;
4186 stringStream ss;
4187 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
4188 b = code_string(ss.as_string());
4189 }
4190 BLOCK_COMMENT("verify_oop {");
4191 #ifdef _LP64
4192 push(rscratch1); // save r10, trashed by movptr()
4193 #endif
4194 push(rax); // save rax,
4195 push(reg); // pass register argument
4196 ExternalAddress buffer((address) b);
4197 // avoid using pushptr, as it modifies scratch registers
4198 // and our contract is not to modify anything
4199 movptr(rax, buffer.addr());
4200 push(rax);
4219 // cf. TemplateTable::prepare_invoke(), if (load_receiver).
4220 int stackElementSize = Interpreter::stackElementSize;
4221 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
4222 #ifdef ASSERT
4223 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
4224 assert(offset1 - offset == stackElementSize, "correct arithmetic");
4225 #endif
4226 Register scale_reg = noreg;
4227 Address::ScaleFactor scale_factor = Address::no_scale;
4228 if (arg_slot.is_constant()) {
4229 offset += arg_slot.as_constant() * stackElementSize;
4230 } else {
4231 scale_reg = arg_slot.as_register();
4232 scale_factor = Address::times(stackElementSize);
4233 }
4234 offset += wordSize; // return PC is on stack
4235 return Address(rsp, scale_reg, scale_factor, offset);
4236 }
4237
4238 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
4239 if (!VerifyOops) return;
4240
4241 // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
4242 // Pass register number to verify_oop_subroutine
4243 const char* b = NULL;
4244 {
4245 ResourceMark rm;
4246 stringStream ss;
4247 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
4248 b = code_string(ss.as_string());
4249 }
4250 #ifdef _LP64
4251 push(rscratch1); // save r10, trashed by movptr()
4252 #endif
4253 push(rax); // save rax,
4254 // addr may contain rsp so we will have to adjust it based on the push
4255 // we just did (and on 64 bit we do two pushes)
4256 // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
4257 // stores rax into addr which is backwards of what was intended.
4258 if (addr.uses(rsp)) {
4259 lea(rax, addr);
4721
4722 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
4723 // get mirror
4724 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
4725 load_method_holder(mirror, method);
4726 movptr(mirror, Address(mirror, mirror_offset));
4727 resolve_oop_handle(mirror, tmp);
4728 }
4729
4730 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4731 load_method_holder(rresult, rmethod);
4732 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4733 }
4734
4735 void MacroAssembler::load_method_holder(Register holder, Register method) {
4736 movptr(holder, Address(method, Method::const_offset())); // ConstMethod*
4737 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
4738 movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4739 }
4740
4741 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
4742 assert_different_registers(src, tmp);
4743 assert_different_registers(dst, tmp);
4744 #ifdef _LP64
4745 if (UseCompressedClassPointers) {
4746 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4747 decode_klass_not_null(dst, tmp);
4748 } else
4749 #endif
4750 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4751 }
4752
4753 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
4754 assert_different_registers(src, tmp);
4755 assert_different_registers(dst, tmp);
4756 #ifdef _LP64
4757 if (UseCompressedClassPointers) {
4758 encode_klass_not_null(src, tmp);
4759 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4760 } else
4761 #endif
4762 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
4763 }
4764
4765 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
4766 Register tmp1, Register thread_tmp) {
4767 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4768 decorators = AccessInternal::decorator_fixup(decorators);
4769 bool as_raw = (decorators & AS_RAW) != 0;
4770 if (as_raw) {
4771 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4772 } else {
4773 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4774 }
4775 }
4776
4777 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
4778 Register tmp1, Register tmp2, Register tmp3) {
4779 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4780 decorators = AccessInternal::decorator_fixup(decorators);
4781 bool as_raw = (decorators & AS_RAW) != 0;
4782 if (as_raw) {
4783 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
4784 } else {
4785 bs->store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
4786 }
4787 }
4788
4789 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
4790 Register thread_tmp, DecoratorSet decorators) {
4791 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
4792 }
4793
4794 // Doesn't do verfication, generates fixed size code
4795 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
4796 Register thread_tmp, DecoratorSet decorators) {
4797 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
4798 }
4799
4800 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
4801 Register tmp2, Register tmp3, DecoratorSet decorators) {
4802 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2, tmp3);
4803 }
4804
4805 // Used for storing NULLs.
4806 void MacroAssembler::store_heap_oop_null(Address dst) {
4807 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
4808 }
5106 RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5107 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5108 }
5109
5110 void MacroAssembler::reinit_heapbase() {
5111 if (UseCompressedOops) {
5112 if (Universe::heap() != NULL) {
5113 if (CompressedOops::base() == NULL) {
5114 MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
5115 } else {
5116 mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
5117 }
5118 } else {
5119 movptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
5120 }
5121 }
5122 }
5123
5124 #endif // _LP64
5125
5126 // C2 compiled method's prolog code.
5127 void MacroAssembler::verified_entry(int framesize, int stack_bang_size, bool fp_mode_24b, bool is_stub) {
5128
5129 // WARNING: Initial instruction MUST be 5 bytes or longer so that
5130 // NativeJump::patch_verified_entry will be able to patch out the entry
5131 // code safely. The push to verify stack depth is ok at 5 bytes,
5132 // the frame allocation can be either 3 or 6 bytes. So if we don't do
5133 // stack bang then we must use the 6 byte frame allocation even if
5134 // we have no frame. :-(
5135 assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
5136
5137 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
5138 // Remove word for return addr
5139 framesize -= wordSize;
5140 stack_bang_size -= wordSize;
5141
5142 // Calls to C2R adapters often do not accept exceptional returns.
5143 // We require that their callers must bang for them. But be careful, because
5144 // some VM calls (such as call site linkage) can use several kilobytes of
5145 // stack. But the stack safety zone should account for that.
5146 // See bugs 4446381, 4468289, 4497237.
5147 if (stack_bang_size > 0) {
5160 // Create frame
5161 if (framesize) {
5162 subptr(rsp, framesize);
5163 }
5164 } else {
5165 // Create frame (force generation of a 4 byte immediate value)
5166 subptr_imm32(rsp, framesize);
5167
5168 // Save RBP register now.
5169 framesize -= wordSize;
5170 movptr(Address(rsp, framesize), rbp);
5171 // Save caller's stack pointer into RBP if the frame pointer is preserved.
5172 if (PreserveFramePointer) {
5173 movptr(rbp, rsp);
5174 if (framesize > 0) {
5175 addptr(rbp, framesize);
5176 }
5177 }
5178 }
5179
5180 if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
5181 framesize -= wordSize;
5182 movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
5183 }
5184
5185 #ifndef _LP64
5186 // If method sets FPU control word do it now
5187 if (fp_mode_24b) {
5188 fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
5189 }
5190 if (UseSSE >= 2 && VerifyFPU) {
5191 verify_FPU(0, "FPU stack must be clean on entry");
5192 }
5193 #endif
5194
5195 #ifdef ASSERT
5196 if (VerifyStackAtCalls) {
5197 Label L;
5198 push(rax);
5199 mov(rax, rsp);
5200 andptr(rax, StackAlignmentInBytes-1);
5201 cmpptr(rax, StackAlignmentInBytes-wordSize);
5202 pop(rax);
5203 jcc(Assembler::equal, L);
5204 STOP("Stack is not properly aligned!");
5205 bind(L);
5206 }
5207 #endif
5208
5209 if (!is_stub) {
5210 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5211 bs->nmethod_entry_barrier(this);
5212 }
5213 }
5214
5215 #if COMPILER2_OR_JVMCI
5216
5217 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
5218 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5219 // cnt - number of qwords (8-byte words).
5220 // base - start address, qword aligned.
5221 Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
5222 bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
5223 if (use64byteVector) {
5224 vpxor(xtmp, xtmp, xtmp, AVX_512bit);
5225 } else if (MaxVectorSize >= 32) {
5226 vpxor(xtmp, xtmp, xtmp, AVX_256bit);
5227 } else {
5228 pxor(xtmp, xtmp);
5229 }
5230 jmp(L_zero_64_bytes);
5231
5232 BIND(L_loop);
5233 if (MaxVectorSize >= 32) {
5234 fill64(base, 0, xtmp, use64byteVector);
5235 } else {
5236 movdqu(Address(base, 0), xtmp);
5237 movdqu(Address(base, 16), xtmp);
5238 movdqu(Address(base, 32), xtmp);
5239 movdqu(Address(base, 48), xtmp);
5240 }
5241 addptr(base, 64);
5242
5243 BIND(L_zero_64_bytes);
5244 subptr(cnt, 8);
5245 jccb(Assembler::greaterEqual, L_loop);
5246
5247 // Copy trailing 64 bytes
5248 if (use64byteVector) {
5249 addptr(cnt, 8);
5250 jccb(Assembler::equal, L_end);
5251 fill64_masked(3, base, 0, xtmp, mask, cnt, rtmp, true);
5252 jmp(L_end);
5253 } else {
5254 addptr(cnt, 4);
5255 jccb(Assembler::less, L_tail);
5256 if (MaxVectorSize >= 32) {
5257 vmovdqu(Address(base, 0), xtmp);
5258 } else {
5259 movdqu(Address(base, 0), xtmp);
5260 movdqu(Address(base, 16), xtmp);
5261 }
5262 }
5263 addptr(base, 32);
5264 subptr(cnt, 4);
5265
5266 BIND(L_tail);
5267 addptr(cnt, 4);
5268 jccb(Assembler::lessEqual, L_end);
5269 if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
5270 fill32_masked(3, base, 0, xtmp, mask, cnt, rtmp);
5271 } else {
5272 decrement(cnt);
5273
5274 BIND(L_sloop);
5275 movq(Address(base, 0), xtmp);
5276 addptr(base, 8);
5277 decrement(cnt);
5278 jccb(Assembler::greaterEqual, L_sloop);
5279 }
5280 BIND(L_end);
5281 }
5282
5283 // Clearing constant sized memory using YMM/ZMM registers.
5284 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
5285 assert(UseAVX > 2 && VM_Version::supports_avx512vlbw(), "");
5286 bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
5287
5288 int vector64_count = (cnt & (~0x7)) >> 3;
5289 cnt = cnt & 0x7;
5290
5291 // 64 byte initialization loop.
5292 vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
5293 for (int i = 0; i < vector64_count; i++) {
5294 fill64(base, i * 64, xtmp, use64byteVector);
5295 }
5296
5297 // Clear remaining 64 byte tail.
5298 int disp = vector64_count * 64;
5299 if (cnt) {
5300 switch (cnt) {
5301 case 1:
5302 movq(Address(base, disp), xtmp);
5334 break;
5335 case 7:
5336 if (use64byteVector) {
5337 movl(rtmp, 0x7F);
5338 kmovwl(mask, rtmp);
5339 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, Assembler::AVX_512bit);
5340 } else {
5341 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, Assembler::AVX_256bit);
5342 movl(rtmp, 0x7);
5343 kmovwl(mask, rtmp);
5344 evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, Assembler::AVX_256bit);
5345 }
5346 break;
5347 default:
5348 fatal("Unexpected length : %d\n",cnt);
5349 break;
5350 }
5351 }
5352 }
5353
5354 void MacroAssembler::clear_mem(Register base, Register cnt, Register tmp, XMMRegister xtmp,
5355 bool is_large, KRegister mask) {
5356 // cnt - number of qwords (8-byte words).
5357 // base - start address, qword aligned.
5358 // is_large - if optimizers know cnt is larger than InitArrayShortSize
5359 assert(base==rdi, "base register must be edi for rep stos");
5360 assert(tmp==rax, "tmp register must be eax for rep stos");
5361 assert(cnt==rcx, "cnt register must be ecx for rep stos");
5362 assert(InitArrayShortSize % BytesPerLong == 0,
5363 "InitArrayShortSize should be the multiple of BytesPerLong");
5364
5365 Label DONE;
5366 if (!is_large || !UseXMMForObjInit) {
5367 xorptr(tmp, tmp);
5368 }
5369
5370 if (!is_large) {
5371 Label LOOP, LONG;
5372 cmpptr(cnt, InitArrayShortSize/BytesPerLong);
5373 jccb(Assembler::greater, LONG);
5374
5375 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
5376
5377 decrement(cnt);
5378 jccb(Assembler::negative, DONE); // Zero length
5379
5380 // Use individual pointer-sized stores for small counts:
5381 BIND(LOOP);
5382 movptr(Address(base, cnt, Address::times_ptr), tmp);
5383 decrement(cnt);
5384 jccb(Assembler::greaterEqual, LOOP);
5385 jmpb(DONE);
5386
5387 BIND(LONG);
5388 }
5389
5390 // Use longer rep-prefixed ops for non-small counts:
5391 if (UseFastStosb) {
5392 shlptr(cnt, 3); // convert to number of bytes
5393 rep_stosb();
5394 } else if (UseXMMForObjInit) {
5395 xmm_clear_mem(base, cnt, tmp, xtmp, mask);
5396 } else {
5397 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
5398 rep_stos();
5399 }
5400
5401 BIND(DONE);
5402 }
5403
5404 #endif //COMPILER2_OR_JVMCI
5405
5406
5407 void MacroAssembler::generate_fill(BasicType t, bool aligned,
5408 Register to, Register value, Register count,
5409 Register rtmp, XMMRegister xtmp) {
5410 ShortBranchVerifier sbv(this);
5411 assert_different_registers(to, value, count, rtmp);
5412 Label L_exit;
5413 Label L_fill_2_bytes, L_fill_4_bytes;
5414
5415 #if defined(COMPILER2) && defined(_LP64)
|
12 * version 2 for more details (a copy is included in the LICENSE file that
13 * accompanied this code).
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "jvm.h"
27 #include "asm/assembler.hpp"
28 #include "asm/assembler.inline.hpp"
29 #include "c1/c1_FrameMap.hpp"
30 #include "compiler/compiler_globals.hpp"
31 #include "compiler/disassembler.hpp"
32 #include "ci/ciInlineKlass.hpp"
33 #include "gc/shared/barrierSet.hpp"
34 #include "gc/shared/barrierSetAssembler.hpp"
35 #include "gc/shared/collectedHeap.inline.hpp"
36 #include "gc/shared/tlab_globals.hpp"
37 #include "interpreter/bytecodeHistogram.hpp"
38 #include "interpreter/interpreter.hpp"
39 #include "memory/resourceArea.hpp"
40 #include "memory/universe.hpp"
41 #include "oops/accessDecorators.hpp"
42 #include "oops/compressedOops.inline.hpp"
43 #include "oops/klass.inline.hpp"
44 #include "prims/methodHandles.hpp"
45 #include "runtime/flags/flagSetting.hpp"
46 #include "runtime/interfaceSupport.inline.hpp"
47 #include "runtime/jniHandles.hpp"
48 #include "runtime/objectMonitor.hpp"
49 #include "runtime/os.hpp"
50 #include "runtime/safepoint.hpp"
51 #include "runtime/safepointMechanism.hpp"
52 #include "runtime/sharedRuntime.hpp"
53 #include "runtime/signature_cc.hpp"
54 #include "runtime/stubRoutines.hpp"
55 #include "runtime/thread.hpp"
56 #include "utilities/macros.hpp"
57 #include "vmreg_x86.inline.hpp"
58 #include "crc32c.h"
59 #ifdef COMPILER2
60 #include "opto/output.hpp"
61 #endif
62
63 #ifdef PRODUCT
64 #define BLOCK_COMMENT(str) /* nothing */
65 #define STOP(error) stop(error)
66 #else
67 #define BLOCK_COMMENT(str) block_comment(str)
68 #define STOP(error) block_comment(error); stop(error)
69 #endif
70
71 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
72
73 #ifdef ASSERT
74 bool AbstractAssembler::pd_check_instruction_mark() { return true; }
75 #endif
76
77 static Assembler::Condition reverse[] = {
78 Assembler::noOverflow /* overflow = 0x0 */ ,
79 Assembler::overflow /* noOverflow = 0x1 */ ,
80 Assembler::aboveEqual /* carrySet = 0x2, below = 0x2 */ ,
81 Assembler::below /* aboveEqual = 0x3, carryClear = 0x3 */ ,
1625 }
1626
1627 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1628
1629 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1630 pass_arg1(this, arg_1);
1631 pass_arg0(this, arg_0);
1632 call_VM_leaf(entry_point, 2);
1633 }
1634
1635 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1636 LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
1637 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1638 pass_arg2(this, arg_2);
1639 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1640 pass_arg1(this, arg_1);
1641 pass_arg0(this, arg_0);
1642 call_VM_leaf(entry_point, 3);
1643 }
1644
1645 void MacroAssembler::super_call_VM_leaf(address entry_point) {
1646 MacroAssembler::call_VM_leaf_base(entry_point, 1);
1647 }
1648
1649 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1650 pass_arg0(this, arg_0);
1651 MacroAssembler::call_VM_leaf_base(entry_point, 1);
1652 }
1653
1654 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1655
1656 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1657 pass_arg1(this, arg_1);
1658 pass_arg0(this, arg_0);
1659 MacroAssembler::call_VM_leaf_base(entry_point, 2);
1660 }
1661
1662 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1663 LP64_ONLY(assert(arg_0 != c_rarg2, "smashed arg"));
1664 LP64_ONLY(assert(arg_1 != c_rarg2, "smashed arg"));
1665 pass_arg2(this, arg_2);
1666 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
1667 pass_arg1(this, arg_1);
1668 pass_arg0(this, arg_0);
2723 lea(rscratch1, src);
2724 Assembler::mulss(dst, Address(rscratch1, 0));
2725 }
2726 }
2727
2728 void MacroAssembler::null_check(Register reg, int offset) {
2729 if (needs_explicit_null_check(offset)) {
2730 // provoke OS NULL exception if reg = NULL by
2731 // accessing M[reg] w/o changing any (non-CC) registers
2732 // NOTE: cmpl is plenty here to provoke a segv
2733 cmpptr(rax, Address(reg, 0));
2734 // Note: should probably use testl(rax, Address(reg, 0));
2735 // may be shorter code (however, this version of
2736 // testl needs to be implemented first)
2737 } else {
2738 // nothing to do, (later) access of M[reg + offset]
2739 // will provoke OS NULL exception if reg = NULL
2740 }
2741 }
2742
2743 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
2744 andptr(markword, markWord::inline_type_mask_in_place);
2745 cmpptr(markword, markWord::inline_type_pattern);
2746 jcc(Assembler::equal, is_inline_type);
2747 }
2748
2749 void MacroAssembler::test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type) {
2750 movl(temp_reg, Address(klass, Klass::access_flags_offset()));
2751 testl(temp_reg, JVM_ACC_VALUE);
2752 jcc(Assembler::notZero, is_inline_type);
2753 }
2754
2755 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type) {
2756 testptr(object, object);
2757 jcc(Assembler::zero, not_inline_type);
2758 const int is_inline_type_mask = markWord::inline_type_pattern;
2759 movptr(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
2760 andptr(tmp, is_inline_type_mask);
2761 cmpptr(tmp, is_inline_type_mask);
2762 jcc(Assembler::notEqual, not_inline_type);
2763 }
2764
2765 void MacroAssembler::test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type) {
2766 #ifdef ASSERT
2767 {
2768 Label done_check;
2769 test_klass_is_inline_type(klass, temp_reg, done_check);
2770 stop("test_klass_is_empty_inline_type with non inline type klass");
2771 bind(done_check);
2772 }
2773 #endif
2774 movl(temp_reg, Address(klass, InstanceKlass::misc_flags_offset()));
2775 testl(temp_reg, InstanceKlass::misc_flag_is_empty_inline_type());
2776 jcc(Assembler::notZero, is_empty_inline_type);
2777 }
2778
2779 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
2780 movl(temp_reg, flags);
2781 shrl(temp_reg, ConstantPoolCacheEntry::is_null_free_inline_type_shift);
2782 andl(temp_reg, 0x1);
2783 testl(temp_reg, temp_reg);
2784 jcc(Assembler::notZero, is_null_free_inline_type);
2785 }
2786
2787 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
2788 movl(temp_reg, flags);
2789 shrl(temp_reg, ConstantPoolCacheEntry::is_null_free_inline_type_shift);
2790 andl(temp_reg, 0x1);
2791 testl(temp_reg, temp_reg);
2792 jcc(Assembler::zero, not_null_free_inline_type);
2793 }
2794
2795 void MacroAssembler::test_field_is_inlined(Register flags, Register temp_reg, Label& is_inlined) {
2796 movl(temp_reg, flags);
2797 shrl(temp_reg, ConstantPoolCacheEntry::is_inlined_shift);
2798 andl(temp_reg, 0x1);
2799 testl(temp_reg, temp_reg);
2800 jcc(Assembler::notZero, is_inlined);
2801 }
2802
2803 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label) {
2804 Label test_mark_word;
2805 // load mark word
2806 movptr(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
2807 // check displaced
2808 testl(temp_reg, markWord::unlocked_value);
2809 jccb(Assembler::notZero, test_mark_word);
2810 // slow path use klass prototype
2811 push(rscratch1);
2812 load_prototype_header(temp_reg, oop, rscratch1);
2813 pop(rscratch1);
2814
2815 bind(test_mark_word);
2816 testl(temp_reg, test_bit);
2817 jcc((jmp_set) ? Assembler::notZero : Assembler::zero, jmp_label);
2818 }
2819
2820 void MacroAssembler::test_flattened_array_oop(Register oop, Register temp_reg,
2821 Label&is_flattened_array) {
2822 #ifdef _LP64
2823 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flattened_array);
2824 #else
2825 load_klass(temp_reg, oop, noreg);
2826 movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
2827 test_flattened_array_layout(temp_reg, is_flattened_array);
2828 #endif
2829 }
2830
2831 void MacroAssembler::test_non_flattened_array_oop(Register oop, Register temp_reg,
2832 Label&is_non_flattened_array) {
2833 #ifdef _LP64
2834 test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flattened_array);
2835 #else
2836 load_klass(temp_reg, oop, noreg);
2837 movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
2838 test_non_flattened_array_layout(temp_reg, is_non_flattened_array);
2839 #endif
2840 }
2841
2842 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label&is_null_free_array) {
2843 #ifdef _LP64
2844 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
2845 #else
2846 load_klass(temp_reg, oop, noreg);
2847 movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
2848 test_null_free_array_layout(temp_reg, is_null_free_array);
2849 #endif
2850 }
2851
2852 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
2853 #ifdef _LP64
2854 test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
2855 #else
2856 load_klass(temp_reg, oop, noreg);
2857 movl(temp_reg, Address(temp_reg, Klass::layout_helper_offset()));
2858 test_non_null_free_array_layout(temp_reg, is_non_null_free_array);
2859 #endif
2860 }
2861
2862 void MacroAssembler::test_flattened_array_layout(Register lh, Label& is_flattened_array) {
2863 testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
2864 jcc(Assembler::notZero, is_flattened_array);
2865 }
2866
2867 void MacroAssembler::test_non_flattened_array_layout(Register lh, Label& is_non_flattened_array) {
2868 testl(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
2869 jcc(Assembler::zero, is_non_flattened_array);
2870 }
2871
2872 void MacroAssembler::test_null_free_array_layout(Register lh, Label& is_null_free_array) {
2873 testl(lh, Klass::_lh_null_free_array_bit_inplace);
2874 jcc(Assembler::notZero, is_null_free_array);
2875 }
2876
2877 void MacroAssembler::test_non_null_free_array_layout(Register lh, Label& is_non_null_free_array) {
2878 testl(lh, Klass::_lh_null_free_array_bit_inplace);
2879 jcc(Assembler::zero, is_non_null_free_array);
2880 }
2881
2882
2883 void MacroAssembler::os_breakpoint() {
2884 // instead of directly emitting a breakpoint, call os:breakpoint for better debugability
2885 // (e.g., MSVC can't call ps() otherwise)
2886 call(RuntimeAddress(CAST_FROM_FN_PTR(address, os::breakpoint)));
2887 }
2888
2889 void MacroAssembler::unimplemented(const char* what) {
2890 const char* buf = NULL;
2891 {
2892 ResourceMark rm;
2893 stringStream ss;
2894 ss.print("unimplemented: %s", what);
2895 buf = code_string(ss.as_string());
2896 }
2897 stop(buf);
2898 }
2899
2900 #ifdef _LP64
2901 #define XSTATE_BV 0x200
2902 #endif
3701 }
3702
3703 // C++ bool manipulation
3704 void MacroAssembler::testbool(Register dst) {
3705 if(sizeof(bool) == 1)
3706 testb(dst, 0xff);
3707 else if(sizeof(bool) == 2) {
3708 // testw implementation needed for two byte bools
3709 ShouldNotReachHere();
3710 } else if(sizeof(bool) == 4)
3711 testl(dst, dst);
3712 else
3713 // unsupported
3714 ShouldNotReachHere();
3715 }
3716
3717 void MacroAssembler::testptr(Register dst, Register src) {
3718 LP64_ONLY(testq(dst, src)) NOT_LP64(testl(dst, src));
3719 }
3720
3721 // Object / value buffer allocation...
3722 //
3723 // Kills klass and rsi on LP64
3724 void MacroAssembler::allocate_instance(Register klass, Register new_obj,
3725 Register t1, Register t2,
3726 bool clear_fields, Label& alloc_failed)
3727 {
3728 Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop;
3729 Register layout_size = t1;
3730 assert(new_obj == rax, "needs to be rax, according to barrier asm eden_allocate");
3731 assert_different_registers(klass, new_obj, t1, t2);
3732
3733 // get instance_size in InstanceKlass (scaled to a count of bytes)
3734 movl(layout_size, Address(klass, Klass::layout_helper_offset()));
3735 // test to see if it has a finalizer or is malformed in some way
3736 testl(layout_size, Klass::_lh_instance_slow_path_bit);
3737 jcc(Assembler::notZero, slow_case_no_pop);
3738
3739 // Allocate the instance:
3740 // If TLAB is enabled:
3741 // Try to allocate in the TLAB.
3742 // If fails, go to the slow path.
3743 // Else If inline contiguous allocations are enabled:
3744 // Try to allocate in eden.
3745 // If fails due to heap end, go to slow path.
3746 //
3747 // If TLAB is enabled OR inline contiguous is enabled:
3748 // Initialize the allocation.
3749 // Exit.
3750 //
3751 // Go to slow path.
3752 const bool allow_shared_alloc =
3753 Universe::heap()->supports_inline_contig_alloc();
3754
3755 push(klass);
3756 const Register thread = LP64_ONLY(r15_thread) NOT_LP64(klass);
3757 #ifndef _LP64
3758 if (UseTLAB || allow_shared_alloc) {
3759 get_thread(thread);
3760 }
3761 #endif // _LP64
3762
3763 if (UseTLAB) {
3764 tlab_allocate(thread, new_obj, layout_size, 0, klass, t2, slow_case);
3765 if (ZeroTLAB || (!clear_fields)) {
3766 // the fields have been already cleared
3767 jmp(initialize_header);
3768 } else {
3769 // initialize both the header and fields
3770 jmp(initialize_object);
3771 }
3772 } else {
3773 // Allocation in the shared Eden, if allowed.
3774 //
3775 eden_allocate(thread, new_obj, layout_size, 0, t2, slow_case);
3776 }
3777
3778 // If UseTLAB or allow_shared_alloc are true, the object is created above and
3779 // there is an initialize need. Otherwise, skip and go to the slow path.
3780 if (UseTLAB || allow_shared_alloc) {
3781 if (clear_fields) {
3782 // The object is initialized before the header. If the object size is
3783 // zero, go directly to the header initialization.
3784 bind(initialize_object);
3785 decrement(layout_size, sizeof(oopDesc));
3786 jcc(Assembler::zero, initialize_header);
3787
3788 // Initialize topmost object field, divide size by 8, check if odd and
3789 // test if zero.
3790 Register zero = klass;
3791 xorl(zero, zero); // use zero reg to clear memory (shorter code)
3792 shrl(layout_size, LogBytesPerLong); // divide by 2*oopSize and set carry flag if odd
3793
3794 #ifdef ASSERT
3795 // make sure instance_size was multiple of 8
3796 Label L;
3797 // Ignore partial flag stall after shrl() since it is debug VM
3798 jcc(Assembler::carryClear, L);
3799 stop("object size is not multiple of 2 - adjust this code");
3800 bind(L);
3801 // must be > 0, no extra check needed here
3802 #endif
3803
3804 // initialize remaining object fields: instance_size was a multiple of 8
3805 {
3806 Label loop;
3807 bind(loop);
3808 movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 1*oopSize), zero);
3809 NOT_LP64(movptr(Address(new_obj, layout_size, Address::times_8, sizeof(oopDesc) - 2*oopSize), zero));
3810 decrement(layout_size);
3811 jcc(Assembler::notZero, loop);
3812 }
3813 } // clear_fields
3814
3815 // initialize object header only.
3816 bind(initialize_header);
3817 pop(klass);
3818 Register mark_word = t2;
3819 movptr(mark_word, Address(klass, Klass::prototype_header_offset()));
3820 movptr(Address(new_obj, oopDesc::mark_offset_in_bytes ()), mark_word);
3821 #ifdef _LP64
3822 xorl(rsi, rsi); // use zero reg to clear memory (shorter code)
3823 store_klass_gap(new_obj, rsi); // zero klass gap for compressed oops
3824 #endif
3825 movptr(t2, klass); // preserve klass
3826 Register tmp_store_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3827 store_klass(new_obj, t2, tmp_store_klass); // src klass reg is potentially compressed
3828
3829 jmp(done);
3830 }
3831
3832 bind(slow_case);
3833 pop(klass);
3834 bind(slow_case_no_pop);
3835 jmp(alloc_failed);
3836
3837 bind(done);
3838 }
3839
3840 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
3841 void MacroAssembler::tlab_allocate(Register thread, Register obj,
3842 Register var_size_in_bytes,
3843 int con_size_in_bytes,
3844 Register t1,
3845 Register t2,
3846 Label& slow_case) {
3847 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
3848 bs->tlab_allocate(this, thread, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
3849 }
3850
3851 RegSet MacroAssembler::call_clobbered_gp_registers() {
3852 RegSet regs;
3853 #ifdef _LP64
3854 regs += RegSet::of(rax, rcx, rdx);
3855 #ifndef WINDOWS
3856 regs += RegSet::of(rsi, rdi);
3857 #endif
3858 regs += RegSet::range(r8, r11);
3859 #else
4081 // clear topmost word (no jump would be needed if conditional assignment worked here)
4082 movptr(Address(address, index, Address::times_8, offset_in_bytes - 0*BytesPerWord), temp);
4083 // index could be 0 now, must check again
4084 jcc(Assembler::zero, done);
4085 bind(even);
4086 }
4087 #endif // !_LP64
4088 // initialize remaining object fields: index is a multiple of 2 now
4089 {
4090 Label loop;
4091 bind(loop);
4092 movptr(Address(address, index, Address::times_8, offset_in_bytes - 1*BytesPerWord), temp);
4093 NOT_LP64(movptr(Address(address, index, Address::times_8, offset_in_bytes - 2*BytesPerWord), temp);)
4094 decrement(index);
4095 jcc(Assembler::notZero, loop);
4096 }
4097
4098 bind(done);
4099 }
4100
4101 void MacroAssembler::get_inline_type_field_klass(Register klass, Register index, Register inline_klass) {
4102 movptr(inline_klass, Address(klass, InstanceKlass::inline_type_field_klasses_offset()));
4103 #ifdef ASSERT
4104 {
4105 Label done;
4106 cmpptr(inline_klass, 0);
4107 jcc(Assembler::notEqual, done);
4108 stop("get_inline_type_field_klass contains no inline klass");
4109 bind(done);
4110 }
4111 #endif
4112 movptr(inline_klass, Address(inline_klass, index, Address::times_ptr));
4113 }
4114
4115 void MacroAssembler::get_default_value_oop(Register inline_klass, Register temp_reg, Register obj) {
4116 #ifdef ASSERT
4117 {
4118 Label done_check;
4119 test_klass_is_inline_type(inline_klass, temp_reg, done_check);
4120 stop("get_default_value_oop from non inline type klass");
4121 bind(done_check);
4122 }
4123 #endif
4124 Register offset = temp_reg;
4125 // Getting the offset of the pre-allocated default value
4126 movptr(offset, Address(inline_klass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())));
4127 movl(offset, Address(offset, in_bytes(InlineKlass::default_value_offset_offset())));
4128
4129 // Getting the mirror
4130 movptr(obj, Address(inline_klass, in_bytes(Klass::java_mirror_offset())));
4131 resolve_oop_handle(obj, inline_klass);
4132
4133 // Getting the pre-allocated default value from the mirror
4134 Address field(obj, offset, Address::times_1);
4135 load_heap_oop(obj, field);
4136 }
4137
4138 void MacroAssembler::get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj) {
4139 #ifdef ASSERT
4140 {
4141 Label done_check;
4142 test_klass_is_empty_inline_type(inline_klass, temp_reg, done_check);
4143 stop("get_empty_value from non-empty inline klass");
4144 bind(done_check);
4145 }
4146 #endif
4147 get_default_value_oop(inline_klass, temp_reg, obj);
4148 }
4149
4150
4151 // Look up the method for a megamorphic invokeinterface call.
4152 // The target method is determined by <intf_klass, itable_index>.
4153 // The receiver klass is in recv_klass.
4154 // On success, the result will be in method_result, and execution falls through.
4155 // On failure, execution transfers to the given label.
4156 void MacroAssembler::lookup_interface_method(Register recv_klass,
4157 Register intf_klass,
4158 RegisterOrConstant itable_index,
4159 Register method_result,
4160 Register scan_temp,
4161 Label& L_no_such_interface,
4162 bool return_method) {
4163 assert_different_registers(recv_klass, intf_klass, scan_temp);
4164 assert_different_registers(method_result, intf_klass, scan_temp);
4165 assert(recv_klass != method_result || !return_method,
4166 "recv_klass can be destroyed when method isn't needed");
4167
4168 assert(itable_index.is_constant() || itable_index.as_register() == method_result,
4169 "caller must use same register for non-constant itable index as for method");
4170
4479 } else {
4480 Label L;
4481 jccb(negate_condition(cc), L);
4482 movl(dst, src);
4483 bind(L);
4484 }
4485 }
4486
4487 void MacroAssembler::cmov32(Condition cc, Register dst, Register src) {
4488 if (VM_Version::supports_cmov()) {
4489 cmovl(cc, dst, src);
4490 } else {
4491 Label L;
4492 jccb(negate_condition(cc), L);
4493 movl(dst, src);
4494 bind(L);
4495 }
4496 }
4497
4498 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
4499 if (!VerifyOops || VerifyAdapterSharing) {
4500 // Below address of the code string confuses VerifyAdapterSharing
4501 // because it may differ between otherwise equivalent adapters.
4502 return;
4503 }
4504
4505 // Pass register number to verify_oop_subroutine
4506 const char* b = NULL;
4507 {
4508 ResourceMark rm;
4509 stringStream ss;
4510 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
4511 b = code_string(ss.as_string());
4512 }
4513 BLOCK_COMMENT("verify_oop {");
4514 #ifdef _LP64
4515 push(rscratch1); // save r10, trashed by movptr()
4516 #endif
4517 push(rax); // save rax,
4518 push(reg); // pass register argument
4519 ExternalAddress buffer((address) b);
4520 // avoid using pushptr, as it modifies scratch registers
4521 // and our contract is not to modify anything
4522 movptr(rax, buffer.addr());
4523 push(rax);
4542 // cf. TemplateTable::prepare_invoke(), if (load_receiver).
4543 int stackElementSize = Interpreter::stackElementSize;
4544 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
4545 #ifdef ASSERT
4546 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
4547 assert(offset1 - offset == stackElementSize, "correct arithmetic");
4548 #endif
4549 Register scale_reg = noreg;
4550 Address::ScaleFactor scale_factor = Address::no_scale;
4551 if (arg_slot.is_constant()) {
4552 offset += arg_slot.as_constant() * stackElementSize;
4553 } else {
4554 scale_reg = arg_slot.as_register();
4555 scale_factor = Address::times(stackElementSize);
4556 }
4557 offset += wordSize; // return PC is on stack
4558 return Address(rsp, scale_reg, scale_factor, offset);
4559 }
4560
4561 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
4562 if (!VerifyOops || VerifyAdapterSharing) {
4563 // Below address of the code string confuses VerifyAdapterSharing
4564 // because it may differ between otherwise equivalent adapters.
4565 return;
4566 }
4567
4568 // Address adjust(addr.base(), addr.index(), addr.scale(), addr.disp() + BytesPerWord);
4569 // Pass register number to verify_oop_subroutine
4570 const char* b = NULL;
4571 {
4572 ResourceMark rm;
4573 stringStream ss;
4574 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
4575 b = code_string(ss.as_string());
4576 }
4577 #ifdef _LP64
4578 push(rscratch1); // save r10, trashed by movptr()
4579 #endif
4580 push(rax); // save rax,
4581 // addr may contain rsp so we will have to adjust it based on the push
4582 // we just did (and on 64 bit we do two pushes)
4583 // NOTE: 64bit seemed to have had a bug in that it did movq(addr, rax); which
4584 // stores rax into addr which is backwards of what was intended.
4585 if (addr.uses(rsp)) {
4586 lea(rax, addr);
5048
5049 void MacroAssembler::load_mirror(Register mirror, Register method, Register tmp) {
5050 // get mirror
5051 const int mirror_offset = in_bytes(Klass::java_mirror_offset());
5052 load_method_holder(mirror, method);
5053 movptr(mirror, Address(mirror, mirror_offset));
5054 resolve_oop_handle(mirror, tmp);
5055 }
5056
5057 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
5058 load_method_holder(rresult, rmethod);
5059 movptr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
5060 }
5061
5062 void MacroAssembler::load_method_holder(Register holder, Register method) {
5063 movptr(holder, Address(method, Method::const_offset())); // ConstMethod*
5064 movptr(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool*
5065 movptr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
5066 }
5067
5068 void MacroAssembler::load_metadata(Register dst, Register src) {
5069 if (UseCompressedClassPointers) {
5070 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5071 } else {
5072 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5073 }
5074 }
5075
5076 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
5077 assert_different_registers(src, tmp);
5078 assert_different_registers(dst, tmp);
5079 #ifdef _LP64
5080 if (UseCompressedClassPointers) {
5081 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5082 decode_klass_not_null(dst, tmp);
5083 } else
5084 #endif
5085 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
5086 }
5087
5088 void MacroAssembler::load_prototype_header(Register dst, Register src, Register tmp) {
5089 load_klass(dst, src, tmp);
5090 movptr(dst, Address(dst, Klass::prototype_header_offset()));
5091 }
5092
5093 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
5094 assert_different_registers(src, tmp);
5095 assert_different_registers(dst, tmp);
5096 #ifdef _LP64
5097 if (UseCompressedClassPointers) {
5098 encode_klass_not_null(src, tmp);
5099 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5100 } else
5101 #endif
5102 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
5103 }
5104
5105 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, Register dst, Address src,
5106 Register tmp1, Register thread_tmp) {
5107 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5108 decorators = AccessInternal::decorator_fixup(decorators);
5109 bool as_raw = (decorators & AS_RAW) != 0;
5110 if (as_raw) {
5111 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
5112 } else {
5113 bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
5114 }
5115 }
5116
5117 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, Address dst, Register src,
5118 Register tmp1, Register tmp2, Register tmp3) {
5119 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5120 decorators = AccessInternal::decorator_fixup(decorators);
5121 bool as_raw = (decorators & AS_RAW) != 0;
5122 if (as_raw) {
5123 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
5124 } else {
5125 bs->store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
5126 }
5127 }
5128
5129 void MacroAssembler::access_value_copy(DecoratorSet decorators, Register src, Register dst,
5130 Register inline_klass) {
5131 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
5132 bs->value_copy(this, decorators, src, dst, inline_klass);
5133 }
5134
5135 void MacroAssembler::first_field_offset(Register inline_klass, Register offset) {
5136 movptr(offset, Address(inline_klass, InstanceKlass::adr_inlineklass_fixed_block_offset()));
5137 movl(offset, Address(offset, InlineKlass::first_field_offset_offset()));
5138 }
5139
5140 void MacroAssembler::data_for_oop(Register oop, Register data, Register inline_klass) {
5141 // ((address) (void*) o) + vk->first_field_offset();
5142 Register offset = (data == oop) ? rscratch1 : data;
5143 first_field_offset(inline_klass, offset);
5144 if (data == oop) {
5145 addptr(data, offset);
5146 } else {
5147 lea(data, Address(oop, offset));
5148 }
5149 }
5150
5151 void MacroAssembler::data_for_value_array_index(Register array, Register array_klass,
5152 Register index, Register data) {
5153 assert(index != rcx, "index needs to shift by rcx");
5154 assert_different_registers(array, array_klass, index);
5155 assert_different_registers(rcx, array, index);
5156
5157 // array->base() + (index << Klass::layout_helper_log2_element_size(lh));
5158 movl(rcx, Address(array_klass, Klass::layout_helper_offset()));
5159
5160 // Klass::layout_helper_log2_element_size(lh)
5161 // (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask;
5162 shrl(rcx, Klass::_lh_log2_element_size_shift);
5163 andl(rcx, Klass::_lh_log2_element_size_mask);
5164 shlptr(index); // index << rcx
5165
5166 lea(data, Address(array, index, Address::times_1, arrayOopDesc::base_offset_in_bytes(T_PRIMITIVE_OBJECT)));
5167 }
5168
5169 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
5170 Register thread_tmp, DecoratorSet decorators) {
5171 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
5172 }
5173
5174 // Doesn't do verfication, generates fixed size code
5175 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
5176 Register thread_tmp, DecoratorSet decorators) {
5177 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
5178 }
5179
5180 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
5181 Register tmp2, Register tmp3, DecoratorSet decorators) {
5182 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2, tmp3);
5183 }
5184
5185 // Used for storing NULLs.
5186 void MacroAssembler::store_heap_oop_null(Address dst) {
5187 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
5188 }
5486 RelocationHolder rspec = metadata_Relocation::spec(klass_index);
5487 Assembler::cmp_narrow_oop(dst, CompressedKlassPointers::encode(k), rspec);
5488 }
5489
5490 void MacroAssembler::reinit_heapbase() {
5491 if (UseCompressedOops) {
5492 if (Universe::heap() != NULL) {
5493 if (CompressedOops::base() == NULL) {
5494 MacroAssembler::xorptr(r12_heapbase, r12_heapbase);
5495 } else {
5496 mov64(r12_heapbase, (int64_t)CompressedOops::ptrs_base());
5497 }
5498 } else {
5499 movptr(r12_heapbase, ExternalAddress((address)CompressedOops::ptrs_base_addr()));
5500 }
5501 }
5502 }
5503
5504 #endif // _LP64
5505
5506 #ifdef COMPILER2
5507 // C2 compiled method's prolog code.
5508 void MacroAssembler::verified_entry(Compile* C, int sp_inc) {
5509 int framesize = C->output()->frame_size_in_bytes();
5510 int bangsize = C->output()->bang_size_in_bytes();
5511 bool fp_mode_24b = false;
5512 int stack_bang_size = C->output()->need_stack_bang(bangsize) ? bangsize : 0;
5513
5514 // WARNING: Initial instruction MUST be 5 bytes or longer so that
5515 // NativeJump::patch_verified_entry will be able to patch out the entry
5516 // code safely. The push to verify stack depth is ok at 5 bytes,
5517 // the frame allocation can be either 3 or 6 bytes. So if we don't do
5518 // stack bang then we must use the 6 byte frame allocation even if
5519 // we have no frame. :-(
5520 assert(stack_bang_size >= framesize || stack_bang_size <= 0, "stack bang size incorrect");
5521
5522 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
5523 // Remove word for return addr
5524 framesize -= wordSize;
5525 stack_bang_size -= wordSize;
5526
5527 // Calls to C2R adapters often do not accept exceptional returns.
5528 // We require that their callers must bang for them. But be careful, because
5529 // some VM calls (such as call site linkage) can use several kilobytes of
5530 // stack. But the stack safety zone should account for that.
5531 // See bugs 4446381, 4468289, 4497237.
5532 if (stack_bang_size > 0) {
5545 // Create frame
5546 if (framesize) {
5547 subptr(rsp, framesize);
5548 }
5549 } else {
5550 // Create frame (force generation of a 4 byte immediate value)
5551 subptr_imm32(rsp, framesize);
5552
5553 // Save RBP register now.
5554 framesize -= wordSize;
5555 movptr(Address(rsp, framesize), rbp);
5556 // Save caller's stack pointer into RBP if the frame pointer is preserved.
5557 if (PreserveFramePointer) {
5558 movptr(rbp, rsp);
5559 if (framesize > 0) {
5560 addptr(rbp, framesize);
5561 }
5562 }
5563 }
5564
5565 if (C->needs_stack_repair()) {
5566 // Save stack increment just below the saved rbp (also account for fixed framesize and rbp)
5567 assert((sp_inc & (StackAlignmentInBytes-1)) == 0, "stack increment not aligned");
5568 movptr(Address(rsp, framesize - wordSize), sp_inc + framesize + wordSize);
5569 }
5570
5571 if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
5572 framesize -= wordSize;
5573 movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
5574 }
5575
5576 #ifndef _LP64
5577 // If method sets FPU control word do it now
5578 if (fp_mode_24b) {
5579 fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_24()));
5580 }
5581 if (UseSSE >= 2 && VerifyFPU) {
5582 verify_FPU(0, "FPU stack must be clean on entry");
5583 }
5584 #endif
5585
5586 #ifdef ASSERT
5587 if (VerifyStackAtCalls) {
5588 Label L;
5589 push(rax);
5590 mov(rax, rsp);
5591 andptr(rax, StackAlignmentInBytes-1);
5592 cmpptr(rax, StackAlignmentInBytes-wordSize);
5593 pop(rax);
5594 jcc(Assembler::equal, L);
5595 STOP("Stack is not properly aligned!");
5596 bind(L);
5597 }
5598 #endif
5599 }
5600 #endif // COMPILER2
5601
5602 #if COMPILER2_OR_JVMCI
5603
5604 // clear memory of size 'cnt' qwords, starting at 'base' using XMM/YMM/ZMM registers
5605 void MacroAssembler::xmm_clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp, KRegister mask) {
5606 // cnt - number of qwords (8-byte words).
5607 // base - start address, qword aligned.
5608 Label L_zero_64_bytes, L_loop, L_sloop, L_tail, L_end;
5609 bool use64byteVector = (MaxVectorSize == 64) && (VM_Version::avx3_threshold() == 0);
5610 if (use64byteVector) {
5611 evpbroadcastq(xtmp, val, AVX_512bit);
5612 } else if (MaxVectorSize >= 32) {
5613 movdq(xtmp, val);
5614 punpcklqdq(xtmp, xtmp);
5615 vinserti128_high(xtmp, xtmp);
5616 } else {
5617 movdq(xtmp, val);
5618 punpcklqdq(xtmp, xtmp);
5619 }
5620 jmp(L_zero_64_bytes);
5621
5622 BIND(L_loop);
5623 if (MaxVectorSize >= 32) {
5624 fill64(base, 0, xtmp, use64byteVector);
5625 } else {
5626 movdqu(Address(base, 0), xtmp);
5627 movdqu(Address(base, 16), xtmp);
5628 movdqu(Address(base, 32), xtmp);
5629 movdqu(Address(base, 48), xtmp);
5630 }
5631 addptr(base, 64);
5632
5633 BIND(L_zero_64_bytes);
5634 subptr(cnt, 8);
5635 jccb(Assembler::greaterEqual, L_loop);
5636
5637 // Copy trailing 64 bytes
5638 if (use64byteVector) {
5639 addptr(cnt, 8);
5640 jccb(Assembler::equal, L_end);
5641 fill64_masked(3, base, 0, xtmp, mask, cnt, val, true);
5642 jmp(L_end);
5643 } else {
5644 addptr(cnt, 4);
5645 jccb(Assembler::less, L_tail);
5646 if (MaxVectorSize >= 32) {
5647 vmovdqu(Address(base, 0), xtmp);
5648 } else {
5649 movdqu(Address(base, 0), xtmp);
5650 movdqu(Address(base, 16), xtmp);
5651 }
5652 }
5653 addptr(base, 32);
5654 subptr(cnt, 4);
5655
5656 BIND(L_tail);
5657 addptr(cnt, 4);
5658 jccb(Assembler::lessEqual, L_end);
5659 if (UseAVX > 2 && MaxVectorSize >= 32 && VM_Version::supports_avx512vl()) {
5660 fill32_masked(3, base, 0, xtmp, mask, cnt, val);
5661 } else {
5662 decrement(cnt);
5663
5664 BIND(L_sloop);
5665 movq(Address(base, 0), xtmp);
5666 addptr(base, 8);
5667 decrement(cnt);
5668 jccb(Assembler::greaterEqual, L_sloop);
5669 }
5670 BIND(L_end);
5671 }
5672
5673 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
5674 assert(InlineTypeReturnedAsFields, "Inline types should never be returned as fields");
5675 // An inline type might be returned. If fields are in registers we
5676 // need to allocate an inline type instance and initialize it with
5677 // the value of the fields.
5678 Label skip;
5679 // We only need a new buffered inline type if a new one is not returned
5680 testptr(rax, 1);
5681 jcc(Assembler::zero, skip);
5682 int call_offset = -1;
5683
5684 #ifdef _LP64
5685 // The following code is similar to allocate_instance but has some slight differences,
5686 // e.g. object size is always not zero, sometimes it's constant; storing klass ptr after
5687 // allocating is not necessary if vk != NULL, etc. allocate_instance is not aware of these.
5688 Label slow_case;
5689 // 1. Try to allocate a new buffered inline instance either from TLAB or eden space
5690 mov(rscratch1, rax); // save rax for slow_case since *_allocate may corrupt it when allocation failed
5691 if (vk != NULL) {
5692 // Called from C1, where the return type is statically known.
5693 movptr(rbx, (intptr_t)vk->get_InlineKlass());
5694 jint obj_size = vk->layout_helper();
5695 assert(obj_size != Klass::_lh_neutral_value, "inline class in return type must have been resolved");
5696 if (UseTLAB) {
5697 tlab_allocate(r15_thread, rax, noreg, obj_size, r13, r14, slow_case);
5698 } else {
5699 eden_allocate(r15_thread, rax, noreg, obj_size, r13, slow_case);
5700 }
5701 } else {
5702 // Call from interpreter. RAX contains ((the InlineKlass* of the return type) | 0x01)
5703 mov(rbx, rax);
5704 andptr(rbx, -2);
5705 movl(r14, Address(rbx, Klass::layout_helper_offset()));
5706 if (UseTLAB) {
5707 tlab_allocate(r15_thread, rax, r14, 0, r13, r14, slow_case);
5708 } else {
5709 eden_allocate(r15_thread, rax, r14, 0, r13, slow_case);
5710 }
5711 }
5712 if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) {
5713 // 2. Initialize buffered inline instance header
5714 Register buffer_obj = rax;
5715 movptr(Address(buffer_obj, oopDesc::mark_offset_in_bytes()), (intptr_t)markWord::inline_type_prototype().value());
5716 xorl(r13, r13);
5717 store_klass_gap(buffer_obj, r13);
5718 if (vk == NULL) {
5719 // store_klass corrupts rbx(klass), so save it in r13 for later use (interpreter case only).
5720 mov(r13, rbx);
5721 }
5722 Register tmp_store_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
5723 store_klass(buffer_obj, rbx, tmp_store_klass);
5724 // 3. Initialize its fields with an inline class specific handler
5725 if (vk != NULL) {
5726 call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint.
5727 } else {
5728 movptr(rbx, Address(r13, InstanceKlass::adr_inlineklass_fixed_block_offset()));
5729 movptr(rbx, Address(rbx, InlineKlass::pack_handler_offset()));
5730 call(rbx);
5731 }
5732 jmp(skip);
5733 }
5734 bind(slow_case);
5735 // We failed to allocate a new inline type, fall back to a runtime
5736 // call. Some oop field may be live in some registers but we can't
5737 // tell. That runtime call will take care of preserving them
5738 // across a GC if there's one.
5739 mov(rax, rscratch1);
5740 #endif
5741
5742 if (from_interpreter) {
5743 super_call_VM_leaf(StubRoutines::store_inline_type_fields_to_buf());
5744 } else {
5745 call(RuntimeAddress(StubRoutines::store_inline_type_fields_to_buf()));
5746 call_offset = offset();
5747 }
5748
5749 bind(skip);
5750 return call_offset;
5751 }
5752
5753 // Move a value between registers/stack slots and update the reg_state
5754 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
5755 assert(from->is_valid() && to->is_valid(), "source and destination must be valid");
5756 if (reg_state[to->value()] == reg_written) {
5757 return true; // Already written
5758 }
5759 if (from != to && bt != T_VOID) {
5760 if (reg_state[to->value()] == reg_readonly) {
5761 return false; // Not yet writable
5762 }
5763 if (from->is_reg()) {
5764 if (to->is_reg()) {
5765 if (from->is_XMMRegister()) {
5766 if (bt == T_DOUBLE) {
5767 movdbl(to->as_XMMRegister(), from->as_XMMRegister());
5768 } else {
5769 assert(bt == T_FLOAT, "must be float");
5770 movflt(to->as_XMMRegister(), from->as_XMMRegister());
5771 }
5772 } else {
5773 movq(to->as_Register(), from->as_Register());
5774 }
5775 } else {
5776 int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
5777 Address to_addr = Address(rsp, st_off);
5778 if (from->is_XMMRegister()) {
5779 if (bt == T_DOUBLE) {
5780 movdbl(to_addr, from->as_XMMRegister());
5781 } else {
5782 assert(bt == T_FLOAT, "must be float");
5783 movflt(to_addr, from->as_XMMRegister());
5784 }
5785 } else {
5786 movq(to_addr, from->as_Register());
5787 }
5788 }
5789 } else {
5790 Address from_addr = Address(rsp, from->reg2stack() * VMRegImpl::stack_slot_size + wordSize);
5791 if (to->is_reg()) {
5792 if (to->is_XMMRegister()) {
5793 if (bt == T_DOUBLE) {
5794 movdbl(to->as_XMMRegister(), from_addr);
5795 } else {
5796 assert(bt == T_FLOAT, "must be float");
5797 movflt(to->as_XMMRegister(), from_addr);
5798 }
5799 } else {
5800 movq(to->as_Register(), from_addr);
5801 }
5802 } else {
5803 int st_off = to->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
5804 movq(r13, from_addr);
5805 movq(Address(rsp, st_off), r13);
5806 }
5807 }
5808 }
5809 // Update register states
5810 reg_state[from->value()] = reg_writable;
5811 reg_state[to->value()] = reg_written;
5812 return true;
5813 }
5814
5815 // Calculate the extra stack space required for packing or unpacking inline
5816 // args and adjust the stack pointer
5817 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
5818 // Two additional slots to account for return address
5819 int sp_inc = (args_on_stack + 2) * VMRegImpl::stack_slot_size;
5820 sp_inc = align_up(sp_inc, StackAlignmentInBytes);
5821 // Save the return address, adjust the stack (make sure it is properly
5822 // 16-byte aligned) and copy the return address to the new top of the stack.
5823 // The stack will be repaired on return (see MacroAssembler::remove_frame).
5824 assert(sp_inc > 0, "sanity");
5825 pop(r13);
5826 subptr(rsp, sp_inc);
5827 push(r13);
5828 return sp_inc;
5829 }
5830
5831 // Read all fields from an inline type buffer and store the field values in registers/stack slots.
5832 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
5833 VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
5834 RegState reg_state[]) {
5835 assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter");
5836 assert(from->is_valid(), "source must be valid");
5837 bool progress = false;
5838 #ifdef ASSERT
5839 const int start_offset = offset();
5840 #endif
5841
5842 Label L_null, L_notNull;
5843 // Don't use r14 as tmp because it's used for spilling (see MacroAssembler::spill_reg_for)
5844 Register tmp1 = r10;
5845 Register tmp2 = r13;
5846 Register fromReg = noreg;
5847 ScalarizedInlineArgsStream stream(sig, sig_index, to, to_count, to_index, -1);
5848 bool done = true;
5849 bool mark_done = true;
5850 VMReg toReg;
5851 BasicType bt;
5852 // Check if argument requires a null check
5853 bool null_check = false;
5854 VMReg nullCheckReg;
5855 while (stream.next(nullCheckReg, bt)) {
5856 if (sig->at(stream.sig_index())._offset == -1) {
5857 null_check = true;
5858 break;
5859 }
5860 }
5861 stream.reset(sig_index, to_index);
5862 while (stream.next(toReg, bt)) {
5863 assert(toReg->is_valid(), "destination must be valid");
5864 int idx = (int)toReg->value();
5865 if (reg_state[idx] == reg_readonly) {
5866 if (idx != from->value()) {
5867 mark_done = false;
5868 }
5869 done = false;
5870 continue;
5871 } else if (reg_state[idx] == reg_written) {
5872 continue;
5873 }
5874 assert(reg_state[idx] == reg_writable, "must be writable");
5875 reg_state[idx] = reg_written;
5876 progress = true;
5877
5878 if (fromReg == noreg) {
5879 if (from->is_reg()) {
5880 fromReg = from->as_Register();
5881 } else {
5882 int st_off = from->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
5883 movq(tmp1, Address(rsp, st_off));
5884 fromReg = tmp1;
5885 }
5886 if (null_check) {
5887 // Nullable inline type argument, emit null check
5888 testptr(fromReg, fromReg);
5889 jcc(Assembler::zero, L_null);
5890 }
5891 }
5892 int off = sig->at(stream.sig_index())._offset;
5893 if (off == -1) {
5894 assert(null_check, "Missing null check at");
5895 if (toReg->is_stack()) {
5896 int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
5897 movq(Address(rsp, st_off), 1);
5898 } else {
5899 movq(toReg->as_Register(), 1);
5900 }
5901 continue;
5902 }
5903 assert(off > 0, "offset in object should be positive");
5904 Address fromAddr = Address(fromReg, off);
5905 if (!toReg->is_XMMRegister()) {
5906 Register dst = toReg->is_stack() ? tmp2 : toReg->as_Register();
5907 if (is_reference_type(bt)) {
5908 load_heap_oop(dst, fromAddr);
5909 } else {
5910 bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN);
5911 load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed);
5912 }
5913 if (toReg->is_stack()) {
5914 int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
5915 movq(Address(rsp, st_off), dst);
5916 }
5917 } else if (bt == T_DOUBLE) {
5918 movdbl(toReg->as_XMMRegister(), fromAddr);
5919 } else {
5920 assert(bt == T_FLOAT, "must be float");
5921 movflt(toReg->as_XMMRegister(), fromAddr);
5922 }
5923 }
5924 if (progress && null_check) {
5925 if (done) {
5926 jmp(L_notNull);
5927 bind(L_null);
5928 // Set IsInit field to zero to signal that the argument is null.
5929 // Also set all oop fields to zero to make the GC happy.
5930 stream.reset(sig_index, to_index);
5931 while (stream.next(toReg, bt)) {
5932 if (sig->at(stream.sig_index())._offset == -1 ||
5933 bt == T_OBJECT || bt == T_ARRAY) {
5934 if (toReg->is_stack()) {
5935 int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
5936 movq(Address(rsp, st_off), 0);
5937 } else {
5938 xorq(toReg->as_Register(), toReg->as_Register());
5939 }
5940 }
5941 }
5942 bind(L_notNull);
5943 } else {
5944 bind(L_null);
5945 }
5946 }
5947
5948 sig_index = stream.sig_index();
5949 to_index = stream.regs_index();
5950
5951 if (mark_done && reg_state[from->value()] != reg_written) {
5952 // This is okay because no one else will write to that slot
5953 reg_state[from->value()] = reg_writable;
5954 }
5955 from_index--;
5956 assert(progress || (start_offset == offset()), "should not emit code");
5957 return done;
5958 }
5959
5960 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
5961 VMRegPair* from, int from_count, int& from_index, VMReg to,
5962 RegState reg_state[], Register val_array) {
5963 assert(sig->at(sig_index)._bt == T_PRIMITIVE_OBJECT, "should be at end delimiter");
5964 assert(to->is_valid(), "destination must be valid");
5965
5966 if (reg_state[to->value()] == reg_written) {
5967 skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
5968 return true; // Already written
5969 }
5970
5971 // TODO 8284443 Isn't it an issue if below code uses r14 as tmp when it contains a spilled value?
5972 // Be careful with r14 because it's used for spilling (see MacroAssembler::spill_reg_for).
5973 Register val_obj_tmp = r11;
5974 Register from_reg_tmp = r14;
5975 Register tmp1 = r10;
5976 Register tmp2 = r13;
5977 Register tmp3 = rbx;
5978 Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register();
5979
5980 assert_different_registers(val_obj_tmp, from_reg_tmp, tmp1, tmp2, tmp3, val_array);
5981
5982 if (reg_state[to->value()] == reg_readonly) {
5983 if (!is_reg_in_unpacked_fields(sig, sig_index, to, from, from_count, from_index)) {
5984 skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
5985 return false; // Not yet writable
5986 }
5987 val_obj = val_obj_tmp;
5988 }
5989
5990 int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_PRIMITIVE_OBJECT);
5991 load_heap_oop(val_obj, Address(val_array, index));
5992
5993 ScalarizedInlineArgsStream stream(sig, sig_index, from, from_count, from_index);
5994 VMReg fromReg;
5995 BasicType bt;
5996 Label L_null;
5997 while (stream.next(fromReg, bt)) {
5998 assert(fromReg->is_valid(), "source must be valid");
5999 reg_state[fromReg->value()] = reg_writable;
6000
6001 int off = sig->at(stream.sig_index())._offset;
6002 if (off == -1) {
6003 // Nullable inline type argument, emit null check
6004 Label L_notNull;
6005 if (fromReg->is_stack()) {
6006 int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
6007 testb(Address(rsp, ld_off), 1);
6008 } else {
6009 testb(fromReg->as_Register(), 1);
6010 }
6011 jcc(Assembler::notZero, L_notNull);
6012 movptr(val_obj, 0);
6013 jmp(L_null);
6014 bind(L_notNull);
6015 continue;
6016 }
6017
6018 assert(off > 0, "offset in object should be positive");
6019 size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
6020
6021 Address dst(val_obj, off);
6022 if (!fromReg->is_XMMRegister()) {
6023 Register src;
6024 if (fromReg->is_stack()) {
6025 src = from_reg_tmp;
6026 int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size + wordSize;
6027 load_sized_value(src, Address(rsp, ld_off), size_in_bytes, /* is_signed */ false);
6028 } else {
6029 src = fromReg->as_Register();
6030 }
6031 assert_different_registers(dst.base(), src, tmp1, tmp2, tmp3, val_array);
6032 if (is_reference_type(bt)) {
6033 store_heap_oop(dst, src, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
6034 } else {
6035 store_sized_value(dst, src, size_in_bytes);
6036 }
6037 } else if (bt == T_DOUBLE) {
6038 movdbl(dst, fromReg->as_XMMRegister());
6039 } else {
6040 assert(bt == T_FLOAT, "must be float");
6041 movflt(dst, fromReg->as_XMMRegister());
6042 }
6043 }
6044 bind(L_null);
6045 sig_index = stream.sig_index();
6046 from_index = stream.regs_index();
6047
6048 assert(reg_state[to->value()] == reg_writable, "must have already been read");
6049 bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state);
6050 assert(success, "to register must be writeable");
6051 return true;
6052 }
6053
6054 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
6055 return reg->is_XMMRegister() ? xmm8->as_VMReg() : r14->as_VMReg();
6056 }
6057
6058 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
6059 assert((initial_framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
6060 if (needs_stack_repair) {
6061 movq(rbp, Address(rsp, initial_framesize));
6062 // The stack increment resides just below the saved rbp
6063 addq(rsp, Address(rsp, initial_framesize - wordSize));
6064 } else {
6065 if (initial_framesize > 0) {
6066 addq(rsp, initial_framesize);
6067 }
6068 pop(rbp);
6069 }
6070 }
6071
6072 // Clearing constant sized memory using YMM/ZMM registers.
6073 void MacroAssembler::clear_mem(Register base, int cnt, Register rtmp, XMMRegister xtmp, KRegister mask) {
6074 assert(UseAVX > 2 && VM_Version::supports_avx512vlbw(), "");
6075 bool use64byteVector = (MaxVectorSize > 32) && (VM_Version::avx3_threshold() == 0);
6076
6077 int vector64_count = (cnt & (~0x7)) >> 3;
6078 cnt = cnt & 0x7;
6079
6080 // 64 byte initialization loop.
6081 vpxor(xtmp, xtmp, xtmp, use64byteVector ? AVX_512bit : AVX_256bit);
6082 for (int i = 0; i < vector64_count; i++) {
6083 fill64(base, i * 64, xtmp, use64byteVector);
6084 }
6085
6086 // Clear remaining 64 byte tail.
6087 int disp = vector64_count * 64;
6088 if (cnt) {
6089 switch (cnt) {
6090 case 1:
6091 movq(Address(base, disp), xtmp);
6123 break;
6124 case 7:
6125 if (use64byteVector) {
6126 movl(rtmp, 0x7F);
6127 kmovwl(mask, rtmp);
6128 evmovdqu(T_LONG, mask, Address(base, disp), xtmp, Assembler::AVX_512bit);
6129 } else {
6130 evmovdqu(T_LONG, k0, Address(base, disp), xtmp, Assembler::AVX_256bit);
6131 movl(rtmp, 0x7);
6132 kmovwl(mask, rtmp);
6133 evmovdqu(T_LONG, mask, Address(base, disp + 32), xtmp, Assembler::AVX_256bit);
6134 }
6135 break;
6136 default:
6137 fatal("Unexpected length : %d\n",cnt);
6138 break;
6139 }
6140 }
6141 }
6142
6143 void MacroAssembler::clear_mem(Register base, Register cnt, Register val, XMMRegister xtmp,
6144 bool is_large, bool word_copy_only, KRegister mask) {
6145 // cnt - number of qwords (8-byte words).
6146 // base - start address, qword aligned.
6147 // is_large - if optimizers know cnt is larger than InitArrayShortSize
6148 assert(base==rdi, "base register must be edi for rep stos");
6149 assert(val==rax, "val register must be eax for rep stos");
6150 assert(cnt==rcx, "cnt register must be ecx for rep stos");
6151 assert(InitArrayShortSize % BytesPerLong == 0,
6152 "InitArrayShortSize should be the multiple of BytesPerLong");
6153
6154 Label DONE;
6155
6156 if (!is_large) {
6157 Label LOOP, LONG;
6158 cmpptr(cnt, InitArrayShortSize/BytesPerLong);
6159 jccb(Assembler::greater, LONG);
6160
6161 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
6162
6163 decrement(cnt);
6164 jccb(Assembler::negative, DONE); // Zero length
6165
6166 // Use individual pointer-sized stores for small counts:
6167 BIND(LOOP);
6168 movptr(Address(base, cnt, Address::times_ptr), val);
6169 decrement(cnt);
6170 jccb(Assembler::greaterEqual, LOOP);
6171 jmpb(DONE);
6172
6173 BIND(LONG);
6174 }
6175
6176 // Use longer rep-prefixed ops for non-small counts:
6177 if (UseFastStosb && !word_copy_only) {
6178 shlptr(cnt, 3); // convert to number of bytes
6179 rep_stosb();
6180 } else if (UseXMMForObjInit) {
6181 xmm_clear_mem(base, cnt, val, xtmp, mask);
6182 } else {
6183 NOT_LP64(shlptr(cnt, 1);) // convert to number of 32-bit words for 32-bit VM
6184 rep_stos();
6185 }
6186
6187 BIND(DONE);
6188 }
6189
6190 #endif //COMPILER2_OR_JVMCI
6191
6192
6193 void MacroAssembler::generate_fill(BasicType t, bool aligned,
6194 Register to, Register value, Register count,
6195 Register rtmp, XMMRegister xtmp) {
6196 ShortBranchVerifier sbv(this);
6197 assert_different_registers(to, value, count, rtmp);
6198 Label L_exit;
6199 Label L_fill_2_bytes, L_fill_4_bytes;
6200
6201 #if defined(COMPILER2) && defined(_LP64)
|