15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/safepointMechanism.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "utilities/powerOfTwo.hpp"
46 #include "vmreg_x86.inline.hpp"
47
48
49 // These masks are used to provide 128-bit aligned bitmasks to the XMM
50 // instructions, to allow sign-masking or sign-bit flipping. They allow
51 // fast versions of NegF/NegD and AbsF/AbsD.
52
53 // Note: 'double' and 'long long' have 32-bits alignment on x86.
54 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
55 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
56 // of 128-bits operands for SSE instructions.
57 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
58 // Store the value to a 128-bits operand.
59 operand[0] = lo;
462 __ bind(*stub->continuation());
463 }
464
465 if (compilation()->env()->dtrace_method_probes()) {
466 #ifdef _LP64
467 __ mov(rdi, r15_thread);
468 __ mov_metadata(rsi, method()->constant_encoding());
469 #else
470 __ get_thread(rax);
471 __ movptr(Address(rsp, 0), rax);
472 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
473 #endif
474 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
475 }
476
477 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
478 __ mov(rax, rbx); // Restore the exception
479 }
480
481 // remove the activation and dispatch to the unwind handler
482 __ remove_frame(initial_frame_size_in_bytes());
483 __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
484
485 // Emit the slow path assembly
486 if (stub != nullptr) {
487 stub->emit_code(this);
488 }
489
490 return offset;
491 }
492
493
494 int LIR_Assembler::emit_deopt_handler() {
495 // generate code for exception handler
496 address handler_base = __ start_a_stub(deopt_handler_size());
497 if (handler_base == nullptr) {
498 // not enough space left for the handler
499 bailout("deopt handler overflow");
500 return -1;
501 }
502
503 int offset = code_offset();
504 InternalAddress here(__ pc());
505
506 __ pushptr(here.addr(), rscratch1);
507 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
508 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
509 __ end_a_stub();
510
511 return offset;
512 }
513
514 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
515 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
516 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
517 assert(result->fpu() == 0, "result must already be on TOS");
518 }
519
520 // Pop the stack before the safepoint code
521 __ remove_frame(initial_frame_size_in_bytes());
522
523 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
524 __ reserved_stack_check();
525 }
526
527 // Note: we do not need to round double result; float result has the right precision
528 // the poll sets the condition code, but no data registers
529
530 #ifdef _LP64
531 const Register thread = r15_thread;
532 #else
533 const Register thread = rbx;
534 __ get_thread(thread);
535 #endif
536 code_stub->set_safepoint_offset(__ offset());
537 __ relocate(relocInfo::poll_return_type);
538 __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
539 __ ret(0);
540 }
541
542
543 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
544 guarantee(info != nullptr, "Shouldn't be null");
545 int offset = __ offset();
546 #ifdef _LP64
547 const Register poll_addr = rscratch1;
548 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
549 #else
550 assert(tmp->is_cpu_register(), "needed");
551 const Register poll_addr = tmp->as_register();
552 __ get_thread(poll_addr);
553 __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
554 #endif
555 add_debug_info_for_branch(info);
556 __ relocate(relocInfo::poll_type);
557 address pre_pc = __ pc();
558 __ testl(rax, Address(poll_addr, 0));
559 address post_pc = __ pc();
560 guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
561 return offset;
562 }
1597 add_debug_info_for_null_check_here(op->stub()->info());
1598 __ cmpb(Address(op->klass()->as_register(),
1599 InstanceKlass::init_state_offset()),
1600 InstanceKlass::fully_initialized);
1601 __ jcc(Assembler::notEqual, *op->stub()->entry());
1602 }
1603 __ allocate_object(op->obj()->as_register(),
1604 op->tmp1()->as_register(),
1605 op->tmp2()->as_register(),
1606 op->header_size(),
1607 op->object_size(),
1608 op->klass()->as_register(),
1609 *op->stub()->entry());
1610 __ bind(*op->stub()->continuation());
1611 }
1612
1613 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1614 Register len = op->len()->as_register();
1615 LP64_ONLY( __ movslq(len, len); )
1616
1617 if (UseSlowPath ||
1618 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1619 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1620 __ jmp(*op->stub()->entry());
1621 } else {
1622 Register tmp1 = op->tmp1()->as_register();
1623 Register tmp2 = op->tmp2()->as_register();
1624 Register tmp3 = op->tmp3()->as_register();
1625 if (len == tmp1) {
1626 tmp1 = tmp3;
1627 } else if (len == tmp2) {
1628 tmp2 = tmp3;
1629 } else if (len == tmp3) {
1630 // everything is ok
1631 } else {
1632 __ mov(tmp3, len);
1633 }
1634 __ allocate_array(op->obj()->as_register(),
1635 len,
1636 tmp1,
1637 tmp2,
1695 assert(data != nullptr, "need data for type check");
1696 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1697 }
1698 Label* success_target = success;
1699 Label* failure_target = failure;
1700
1701 if (obj == k_RInfo) {
1702 k_RInfo = dst;
1703 } else if (obj == klass_RInfo) {
1704 klass_RInfo = dst;
1705 }
1706 if (k->is_loaded() && !UseCompressedClassPointers) {
1707 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1708 } else {
1709 Rtmp1 = op->tmp3()->as_register();
1710 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1711 }
1712
1713 assert_different_registers(obj, k_RInfo, klass_RInfo);
1714
1715 __ testptr(obj, obj);
1716 if (op->should_profile()) {
1717 Label not_null;
1718 Register mdo = klass_RInfo;
1719 __ mov_metadata(mdo, md->constant_encoding());
1720 __ jccb(Assembler::notEqual, not_null);
1721 // Object is null; update MDO and exit
1722 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1723 int header_bits = BitData::null_seen_byte_constant();
1724 __ orb(data_addr, header_bits);
1725 __ jmp(*obj_is_null);
1726 __ bind(not_null);
1727
1728 Label update_done;
1729 Register recv = k_RInfo;
1730 __ load_klass(recv, obj, tmp_load_klass);
1731 type_profile_helper(mdo, md, data, recv, &update_done);
1732
1733 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1734 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1735
1736 __ bind(update_done);
1737 } else {
1738 __ jcc(Assembler::equal, *obj_is_null);
1739 }
1740
1741 if (!k->is_loaded()) {
1742 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1743 } else {
1744 #ifdef _LP64
1745 __ mov_metadata(k_RInfo, k->constant_encoding());
1746 #endif // _LP64
1747 }
1748 __ verify_oop(obj);
1749
1750 if (op->fast_check()) {
1751 // get object class
1752 // not a safepoint as obj null check happens earlier
1753 #ifdef _LP64
1754 if (UseCompressedClassPointers) {
1755 __ load_klass(Rtmp1, obj, tmp_load_klass);
1756 __ cmpptr(k_RInfo, Rtmp1);
1757 } else {
1758 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1910 __ mov(dst, obj);
1911 }
1912 } else
1913 if (code == lir_instanceof) {
1914 Register obj = op->object()->as_register();
1915 Register dst = op->result_opr()->as_register();
1916 Label success, failure, done;
1917 emit_typecheck_helper(op, &success, &failure, &failure);
1918 __ bind(failure);
1919 __ xorptr(dst, dst);
1920 __ jmpb(done);
1921 __ bind(success);
1922 __ movptr(dst, 1);
1923 __ bind(done);
1924 } else {
1925 ShouldNotReachHere();
1926 }
1927
1928 }
1929
1930
1931 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1932 if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) {
1933 assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1934 assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1935 assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1936 assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1937 Register addr = op->addr()->as_register();
1938 __ lock();
1939 NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1940
1941 } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1942 NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1943 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1944 Register newval = op->new_value()->as_register();
1945 Register cmpval = op->cmp_value()->as_register();
1946 assert(cmpval == rax, "wrong register");
1947 assert(newval != noreg, "new val must be register");
1948 assert(cmpval != newval, "cmp and new values must be in different registers");
1949 assert(cmpval != addr, "cmp and addr must be in different registers");
1970 __ cmpxchgl(newval, Address(addr, 0));
1971 }
1972 #ifdef _LP64
1973 } else if (op->code() == lir_cas_long) {
1974 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1975 Register newval = op->new_value()->as_register_lo();
1976 Register cmpval = op->cmp_value()->as_register_lo();
1977 assert(cmpval == rax, "wrong register");
1978 assert(newval != noreg, "new val must be register");
1979 assert(cmpval != newval, "cmp and new values must be in different registers");
1980 assert(cmpval != addr, "cmp and addr must be in different registers");
1981 assert(newval != addr, "new value and addr must be in different registers");
1982 __ lock();
1983 __ cmpxchgq(newval, Address(addr, 0));
1984 #endif // _LP64
1985 } else {
1986 Unimplemented();
1987 }
1988 }
1989
1990 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1991 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1992 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1993
1994 Assembler::Condition acond, ncond;
1995 switch (condition) {
1996 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1997 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1998 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1999 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
2000 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
2001 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
2002 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
2003 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
2004 default: acond = Assembler::equal; ncond = Assembler::notEqual;
2005 ShouldNotReachHere();
2006 }
2007
2008 if (opr1->is_cpu_register()) {
2009 reg2reg(opr1, result);
2847 int offset = __ offset();
2848 switch (code) {
2849 case lir_static_call:
2850 case lir_optvirtual_call:
2851 case lir_dynamic_call:
2852 offset += NativeCall::displacement_offset;
2853 break;
2854 case lir_icvirtual_call:
2855 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size;
2856 break;
2857 default: ShouldNotReachHere();
2858 }
2859 __ align(BytesPerWord, offset);
2860 }
2861
2862
2863 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2864 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2865 "must be aligned");
2866 __ call(AddressLiteral(op->addr(), rtype));
2867 add_call_info(code_offset(), op->info());
2868 __ post_call_nop();
2869 }
2870
2871
2872 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2873 __ ic_call(op->addr());
2874 add_call_info(code_offset(), op->info());
2875 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2876 "must be aligned");
2877 __ post_call_nop();
2878 }
2879
2880
2881 void LIR_Assembler::emit_static_call_stub() {
2882 address call_pc = __ pc();
2883 address stub = __ start_a_stub(call_stub_size());
2884 if (stub == nullptr) {
2885 bailout("static call stub overflow");
2886 return;
2887 }
2888
2889 int start = __ offset();
2890
2891 // make sure that the displacement word of the call ends up word aligned
2892 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset);
2893 __ relocate(static_stub_Relocation::spec(call_pc));
2894 __ mov_metadata(rbx, (Metadata*)nullptr);
3035 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3036 }
3037
3038
3039 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
3040 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3041 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3042 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3043 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
3044 }
3045
3046
3047 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
3048 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3049 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3050 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3051 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
3052 }
3053
3054
3055 // This code replaces a call to arraycopy; no exception may
3056 // be thrown in this code, they must be thrown in the System.arraycopy
3057 // activation frame; we could save some checks if this would not be the case
3058 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3059 ciArrayKlass* default_type = op->expected_type();
3060 Register src = op->src()->as_register();
3061 Register dst = op->dst()->as_register();
3062 Register src_pos = op->src_pos()->as_register();
3063 Register dst_pos = op->dst_pos()->as_register();
3064 Register length = op->length()->as_register();
3065 Register tmp = op->tmp()->as_register();
3066 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3067
3068 CodeStub* stub = op->stub();
3069 int flags = op->flags();
3070 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
3071 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
3072
3073 // if we don't know anything, just go through the generic arraycopy
3074 if (default_type == nullptr) {
3075 // save outgoing arguments on stack in case call to System.arraycopy is needed
3076 // HACK ALERT. This code used to push the parameters in a hardwired fashion
3077 // for interpreter calling conventions. Now we have to do it in new style conventions.
3078 // For the moment until C1 gets the new register allocator I just force all the
3079 // args to the right place (except the register args) and then on the back side
3080 // reload the register args properly if we go slow path. Yuck
3081
3082 // These are proper for the calling convention
3083 store_parameter(length, 2);
3084 store_parameter(dst_pos, 1);
3085 store_parameter(dst, 0);
3086
3087 // these are just temporary placements until we need to reload
3088 store_parameter(src_pos, 3);
3089 store_parameter(src, 4);
3090 NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3091
3092 address copyfunc_addr = StubRoutines::generic_arraycopy();
3146 __ mov(tmp, rax);
3147 __ xorl(tmp, -1);
3148
3149 // Reload values from the stack so they are where the stub
3150 // expects them.
3151 __ movptr (dst, Address(rsp, 0*BytesPerWord));
3152 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
3153 __ movptr (length, Address(rsp, 2*BytesPerWord));
3154 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
3155 __ movptr (src, Address(rsp, 4*BytesPerWord));
3156
3157 __ subl(length, tmp);
3158 __ addl(src_pos, tmp);
3159 __ addl(dst_pos, tmp);
3160 __ jmp(*stub->entry());
3161
3162 __ bind(*stub->continuation());
3163 return;
3164 }
3165
3166 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3167
3168 int elem_size = type2aelembytes(basic_type);
3169 Address::ScaleFactor scale;
3170
3171 switch (elem_size) {
3172 case 1 :
3173 scale = Address::times_1;
3174 break;
3175 case 2 :
3176 scale = Address::times_2;
3177 break;
3178 case 4 :
3179 scale = Address::times_4;
3180 break;
3181 case 8 :
3182 scale = Address::times_8;
3183 break;
3184 default:
3185 scale = Address::no_scale;
3747 __ jccb(Assembler::zero, next);
3748 #endif
3749 // first time here. Set profile type.
3750 __ movptr(mdo_addr, tmp);
3751 } else {
3752 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3753 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3754
3755 __ movptr(tmp, mdo_addr);
3756 __ testptr(tmp, TypeEntries::type_unknown);
3757 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3758
3759 __ orptr(mdo_addr, TypeEntries::type_unknown);
3760 }
3761 }
3762
3763 __ bind(next);
3764 }
3765 }
3766
3767 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3768 Unimplemented();
3769 }
3770
3771
3772 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3773 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3774 }
3775
3776
3777 void LIR_Assembler::align_backward_branch_target() {
3778 __ align(BytesPerWord);
3779 }
3780
3781
3782 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3783 if (left->is_single_cpu()) {
3784 __ negl(left->as_register());
3785 move_regs(left->as_register(), dest->as_register());
3786
4010 }
4011
4012 void LIR_Assembler::membar_storeload() {
4013 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
4014 }
4015
4016 void LIR_Assembler::on_spin_wait() {
4017 __ pause ();
4018 }
4019
4020 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
4021 assert(result_reg->is_register(), "check");
4022 #ifdef _LP64
4023 // __ get_thread(result_reg->as_register_lo());
4024 __ mov(result_reg->as_register(), r15_thread);
4025 #else
4026 __ get_thread(result_reg->as_register());
4027 #endif // _LP64
4028 }
4029
4030
4031 void LIR_Assembler::peephole(LIR_List*) {
4032 // do nothing for now
4033 }
4034
4035 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
4036 assert(data == dest, "xchg/xadd uses only 2 operands");
4037
4038 if (data->type() == T_INT) {
4039 if (code == lir_xadd) {
4040 __ lock();
4041 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
4042 } else {
4043 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
4044 }
4045 } else if (data->is_oop()) {
4046 assert (code == lir_xchg, "xadd for oops");
4047 Register obj = data->as_register();
4048 #ifdef _LP64
4049 if (UseCompressedOops) {
|
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInlineKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
464 __ bind(*stub->continuation());
465 }
466
467 if (compilation()->env()->dtrace_method_probes()) {
468 #ifdef _LP64
469 __ mov(rdi, r15_thread);
470 __ mov_metadata(rsi, method()->constant_encoding());
471 #else
472 __ get_thread(rax);
473 __ movptr(Address(rsp, 0), rax);
474 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
475 #endif
476 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
477 }
478
479 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
480 __ mov(rax, rbx); // Restore the exception
481 }
482
483 // remove the activation and dispatch to the unwind handler
484 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
485 __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
486
487 // Emit the slow path assembly
488 if (stub != nullptr) {
489 stub->emit_code(this);
490 }
491
492 return offset;
493 }
494
495
496 int LIR_Assembler::emit_deopt_handler() {
497 // generate code for exception handler
498 address handler_base = __ start_a_stub(deopt_handler_size());
499 if (handler_base == nullptr) {
500 // not enough space left for the handler
501 bailout("deopt handler overflow");
502 return -1;
503 }
504
505 int offset = code_offset();
506 InternalAddress here(__ pc());
507
508 __ pushptr(here.addr(), rscratch1);
509 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
510 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
511 __ end_a_stub();
512
513 return offset;
514 }
515
516 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
517 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
518 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
519 assert(result->fpu() == 0, "result must already be on TOS");
520 }
521 if (InlineTypeReturnedAsFields) {
522 #ifndef _LP64
523 Unimplemented();
524 #endif
525 // Check if we are returning an non-null inline type and load its fields into registers
526 ciType* return_type = compilation()->method()->return_type();
527 if (return_type->is_inlinetype()) {
528 ciInlineKlass* vk = return_type->as_inline_klass();
529 if (vk->can_be_returned_as_fields()) {
530 address unpack_handler = vk->unpack_handler();
531 assert(unpack_handler != nullptr, "must be");
532 __ call(RuntimeAddress(unpack_handler));
533 }
534 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
535 Label skip;
536 __ test_oop_is_not_inline_type(rax, rscratch1, skip);
537
538 // Load fields from a buffered value with an inline class specific handler
539 __ load_klass(rdi, rax, rscratch1);
540 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
541 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
542 // Unpack handler can be null if inline type is not scalarizable in returns
543 __ testptr(rdi, rdi);
544 __ jcc(Assembler::zero, skip);
545 __ call(rdi);
546
547 __ bind(skip);
548 }
549 // At this point, rax points to the value object (for interpreter or C1 caller).
550 // The fields of the object are copied into registers (for C2 caller).
551 }
552
553 // Pop the stack before the safepoint code
554 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
555
556 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
557 __ reserved_stack_check();
558 }
559
560 // Note: we do not need to round double result; float result has the right precision
561 // the poll sets the condition code, but no data registers
562
563 #ifdef _LP64
564 const Register thread = r15_thread;
565 #else
566 const Register thread = rbx;
567 __ get_thread(thread);
568 #endif
569 code_stub->set_safepoint_offset(__ offset());
570 __ relocate(relocInfo::poll_return_type);
571 __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
572 __ ret(0);
573 }
574
575
576 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
577 return (__ store_inline_type_fields_to_buf(vk, false));
578 }
579
580 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
581 guarantee(info != nullptr, "Shouldn't be null");
582 int offset = __ offset();
583 #ifdef _LP64
584 const Register poll_addr = rscratch1;
585 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
586 #else
587 assert(tmp->is_cpu_register(), "needed");
588 const Register poll_addr = tmp->as_register();
589 __ get_thread(poll_addr);
590 __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
591 #endif
592 add_debug_info_for_branch(info);
593 __ relocate(relocInfo::poll_type);
594 address pre_pc = __ pc();
595 __ testl(rax, Address(poll_addr, 0));
596 address post_pc = __ pc();
597 guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
598 return offset;
599 }
1634 add_debug_info_for_null_check_here(op->stub()->info());
1635 __ cmpb(Address(op->klass()->as_register(),
1636 InstanceKlass::init_state_offset()),
1637 InstanceKlass::fully_initialized);
1638 __ jcc(Assembler::notEqual, *op->stub()->entry());
1639 }
1640 __ allocate_object(op->obj()->as_register(),
1641 op->tmp1()->as_register(),
1642 op->tmp2()->as_register(),
1643 op->header_size(),
1644 op->object_size(),
1645 op->klass()->as_register(),
1646 *op->stub()->entry());
1647 __ bind(*op->stub()->continuation());
1648 }
1649
1650 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1651 Register len = op->len()->as_register();
1652 LP64_ONLY( __ movslq(len, len); )
1653
1654 if (UseSlowPath || op->is_null_free() ||
1655 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1656 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1657 __ jmp(*op->stub()->entry());
1658 } else {
1659 Register tmp1 = op->tmp1()->as_register();
1660 Register tmp2 = op->tmp2()->as_register();
1661 Register tmp3 = op->tmp3()->as_register();
1662 if (len == tmp1) {
1663 tmp1 = tmp3;
1664 } else if (len == tmp2) {
1665 tmp2 = tmp3;
1666 } else if (len == tmp3) {
1667 // everything is ok
1668 } else {
1669 __ mov(tmp3, len);
1670 }
1671 __ allocate_array(op->obj()->as_register(),
1672 len,
1673 tmp1,
1674 tmp2,
1732 assert(data != nullptr, "need data for type check");
1733 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1734 }
1735 Label* success_target = success;
1736 Label* failure_target = failure;
1737
1738 if (obj == k_RInfo) {
1739 k_RInfo = dst;
1740 } else if (obj == klass_RInfo) {
1741 klass_RInfo = dst;
1742 }
1743 if (k->is_loaded() && !UseCompressedClassPointers) {
1744 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1745 } else {
1746 Rtmp1 = op->tmp3()->as_register();
1747 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1748 }
1749
1750 assert_different_registers(obj, k_RInfo, klass_RInfo);
1751
1752 if (op->need_null_check()) {
1753 __ testptr(obj, obj);
1754 if (op->should_profile()) {
1755 Label not_null;
1756 Register mdo = klass_RInfo;
1757 __ mov_metadata(mdo, md->constant_encoding());
1758 __ jccb(Assembler::notEqual, not_null);
1759 // Object is null; update MDO and exit
1760 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1761 int header_bits = BitData::null_seen_byte_constant();
1762 __ orb(data_addr, header_bits);
1763 __ jmp(*obj_is_null);
1764 __ bind(not_null);
1765
1766 Label update_done;
1767 Register recv = k_RInfo;
1768 __ load_klass(recv, obj, tmp_load_klass);
1769 type_profile_helper(mdo, md, data, recv, &update_done);
1770
1771 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1772 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1773
1774 __ bind(update_done);
1775 } else {
1776 __ jcc(Assembler::equal, *obj_is_null);
1777 }
1778 }
1779
1780 if (!k->is_loaded()) {
1781 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1782 } else {
1783 #ifdef _LP64
1784 __ mov_metadata(k_RInfo, k->constant_encoding());
1785 #endif // _LP64
1786 }
1787 __ verify_oop(obj);
1788
1789 if (op->fast_check()) {
1790 // get object class
1791 // not a safepoint as obj null check happens earlier
1792 #ifdef _LP64
1793 if (UseCompressedClassPointers) {
1794 __ load_klass(Rtmp1, obj, tmp_load_klass);
1795 __ cmpptr(k_RInfo, Rtmp1);
1796 } else {
1797 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1949 __ mov(dst, obj);
1950 }
1951 } else
1952 if (code == lir_instanceof) {
1953 Register obj = op->object()->as_register();
1954 Register dst = op->result_opr()->as_register();
1955 Label success, failure, done;
1956 emit_typecheck_helper(op, &success, &failure, &failure);
1957 __ bind(failure);
1958 __ xorptr(dst, dst);
1959 __ jmpb(done);
1960 __ bind(success);
1961 __ movptr(dst, 1);
1962 __ bind(done);
1963 } else {
1964 ShouldNotReachHere();
1965 }
1966
1967 }
1968
1969 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1970 // We are loading/storing from/to an array that *may* be a flat array (the
1971 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1972 // If this array is a flat array, take the slow path.
1973 Register klass = op->tmp()->as_register();
1974 if (UseArrayMarkWordCheck) {
1975 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1976 } else {
1977 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1978 __ load_klass(klass, op->array()->as_register(), tmp_load_klass);
1979 __ movl(klass, Address(klass, Klass::layout_helper_offset()));
1980 __ testl(klass, Klass::_lh_array_tag_flat_value_bit_inplace);
1981 __ jcc(Assembler::notZero, *op->stub()->entry());
1982 }
1983 if (!op->value()->is_illegal()) {
1984 // The array is not a flat array, but it might be null-free. If we are storing
1985 // a null into a null-free array, take the slow path (which will throw NPE).
1986 Label skip;
1987 __ cmpptr(op->value()->as_register(), NULL_WORD);
1988 __ jcc(Assembler::notEqual, skip);
1989 if (UseArrayMarkWordCheck) {
1990 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1991 } else {
1992 __ testl(klass, Klass::_lh_null_free_array_bit_inplace);
1993 __ jcc(Assembler::notZero, *op->stub()->entry());
1994 }
1995 __ bind(skip);
1996 }
1997 }
1998
1999 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
2000 // We are storing into an array that *may* be null-free (the declared type is
2001 // Object[], abstract[], interface[] or VT.ref[]).
2002 if (UseArrayMarkWordCheck) {
2003 Label test_mark_word;
2004 Register tmp = op->tmp()->as_register();
2005 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
2006 __ testl(tmp, markWord::unlocked_value);
2007 __ jccb(Assembler::notZero, test_mark_word);
2008 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
2009 __ bind(test_mark_word);
2010 __ testl(tmp, markWord::null_free_array_bit_in_place);
2011 } else {
2012 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
2013 Register klass = op->tmp()->as_register();
2014 __ load_klass(klass, op->array()->as_register(), tmp_load_klass);
2015 __ movl(klass, Address(klass, Klass::layout_helper_offset()));
2016 __ testl(klass, Klass::_lh_null_free_array_bit_inplace);
2017 }
2018 }
2019
2020 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
2021 Label L_oops_equal;
2022 Label L_oops_not_equal;
2023 Label L_end;
2024
2025 Register left = op->left()->as_register();
2026 Register right = op->right()->as_register();
2027
2028 __ cmpptr(left, right);
2029 __ jcc(Assembler::equal, L_oops_equal);
2030
2031 // (1) Null check -- if one of the operands is null, the other must not be null (because
2032 // the two references are not equal), so they are not substitutable,
2033 // FIXME: do null check only if the operand is nullable
2034 __ testptr(left, right);
2035 __ jcc(Assembler::zero, L_oops_not_equal);
2036
2037 ciKlass* left_klass = op->left_klass();
2038 ciKlass* right_klass = op->right_klass();
2039
2040 // (2) Inline type check -- if either of the operands is not a inline type,
2041 // they are not substitutable. We do this only if we are not sure that the
2042 // operands are inline type
2043 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
2044 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
2045 Register tmp1 = op->tmp1()->as_register();
2046 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
2047 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
2048 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
2049 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
2050 __ jcc(Assembler::notEqual, L_oops_not_equal);
2051 }
2052
2053 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
2054 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
2055 // No need to load klass -- the operands are statically known to be the same inline klass.
2056 __ jmp(*op->stub()->entry());
2057 } else {
2058 Register left_klass_op = op->left_klass_op()->as_register();
2059 Register right_klass_op = op->right_klass_op()->as_register();
2060
2061 if (UseCompressedClassPointers) {
2062 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
2063 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2064 __ cmpl(left_klass_op, right_klass_op);
2065 } else {
2066 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
2067 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2068 __ cmpptr(left_klass_op, right_klass_op);
2069 }
2070
2071 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
2072 // fall through to L_oops_not_equal
2073 }
2074
2075 __ bind(L_oops_not_equal);
2076 move(op->not_equal_result(), op->result_opr());
2077 __ jmp(L_end);
2078
2079 __ bind(L_oops_equal);
2080 move(op->equal_result(), op->result_opr());
2081 __ jmp(L_end);
2082
2083 // We've returned from the stub. RAX contains 0x0 IFF the two
2084 // operands are not substitutable. (Don't compare against 0x1 in case the
2085 // C compiler is naughty)
2086 __ bind(*op->stub()->continuation());
2087 __ cmpl(rax, 0);
2088 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
2089 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
2090 // fall-through
2091 __ bind(L_end);
2092 }
2093
2094 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2095 if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) {
2096 assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
2097 assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
2098 assert(op->new_value()->as_register_lo() == rbx, "wrong register");
2099 assert(op->new_value()->as_register_hi() == rcx, "wrong register");
2100 Register addr = op->addr()->as_register();
2101 __ lock();
2102 NOT_LP64(__ cmpxchg8(Address(addr, 0)));
2103
2104 } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
2105 NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
2106 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2107 Register newval = op->new_value()->as_register();
2108 Register cmpval = op->cmp_value()->as_register();
2109 assert(cmpval == rax, "wrong register");
2110 assert(newval != noreg, "new val must be register");
2111 assert(cmpval != newval, "cmp and new values must be in different registers");
2112 assert(cmpval != addr, "cmp and addr must be in different registers");
2133 __ cmpxchgl(newval, Address(addr, 0));
2134 }
2135 #ifdef _LP64
2136 } else if (op->code() == lir_cas_long) {
2137 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2138 Register newval = op->new_value()->as_register_lo();
2139 Register cmpval = op->cmp_value()->as_register_lo();
2140 assert(cmpval == rax, "wrong register");
2141 assert(newval != noreg, "new val must be register");
2142 assert(cmpval != newval, "cmp and new values must be in different registers");
2143 assert(cmpval != addr, "cmp and addr must be in different registers");
2144 assert(newval != addr, "new value and addr must be in different registers");
2145 __ lock();
2146 __ cmpxchgq(newval, Address(addr, 0));
2147 #endif // _LP64
2148 } else {
2149 Unimplemented();
2150 }
2151 }
2152
2153 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
2154 assert(dst->is_cpu_register(), "must be");
2155 assert(dst->type() == src->type(), "must be");
2156
2157 if (src->is_cpu_register()) {
2158 reg2reg(src, dst);
2159 } else if (src->is_stack()) {
2160 stack2reg(src, dst, dst->type());
2161 } else if (src->is_constant()) {
2162 const2reg(src, dst, lir_patch_none, nullptr);
2163 } else {
2164 ShouldNotReachHere();
2165 }
2166 }
2167
2168 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
2169 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
2170 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
2171
2172 Assembler::Condition acond, ncond;
2173 switch (condition) {
2174 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
2175 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
2176 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
2177 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
2178 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
2179 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
2180 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
2181 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
2182 default: acond = Assembler::equal; ncond = Assembler::notEqual;
2183 ShouldNotReachHere();
2184 }
2185
2186 if (opr1->is_cpu_register()) {
2187 reg2reg(opr1, result);
3025 int offset = __ offset();
3026 switch (code) {
3027 case lir_static_call:
3028 case lir_optvirtual_call:
3029 case lir_dynamic_call:
3030 offset += NativeCall::displacement_offset;
3031 break;
3032 case lir_icvirtual_call:
3033 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size;
3034 break;
3035 default: ShouldNotReachHere();
3036 }
3037 __ align(BytesPerWord, offset);
3038 }
3039
3040
3041 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
3042 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
3043 "must be aligned");
3044 __ call(AddressLiteral(op->addr(), rtype));
3045 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
3046 __ post_call_nop();
3047 }
3048
3049
3050 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
3051 __ ic_call(op->addr());
3052 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
3053 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
3054 "must be aligned");
3055 __ post_call_nop();
3056 }
3057
3058
3059 void LIR_Assembler::emit_static_call_stub() {
3060 address call_pc = __ pc();
3061 address stub = __ start_a_stub(call_stub_size());
3062 if (stub == nullptr) {
3063 bailout("static call stub overflow");
3064 return;
3065 }
3066
3067 int start = __ offset();
3068
3069 // make sure that the displacement word of the call ends up word aligned
3070 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset);
3071 __ relocate(static_stub_Relocation::spec(call_pc));
3072 __ mov_metadata(rbx, (Metadata*)nullptr);
3213 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3214 }
3215
3216
3217 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
3218 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3219 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3220 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3221 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
3222 }
3223
3224
3225 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
3226 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3227 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3228 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3229 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
3230 }
3231
3232
3233 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
3234 if (null_check) {
3235 __ testptr(obj, obj);
3236 __ jcc(Assembler::zero, *slow_path->entry());
3237 }
3238 if (UseArrayMarkWordCheck) {
3239 if (is_dest) {
3240 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
3241 } else {
3242 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
3243 }
3244 } else {
3245 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3246 __ load_klass(tmp, obj, tmp_load_klass);
3247 __ movl(tmp, Address(tmp, Klass::layout_helper_offset()));
3248 if (is_dest) {
3249 // Take the slow path if it's a null_free destination array, in case the source array contains nullptrs.
3250 __ testl(tmp, Klass::_lh_null_free_array_bit_inplace);
3251 } else {
3252 __ testl(tmp, Klass::_lh_array_tag_flat_value_bit_inplace);
3253 }
3254 __ jcc(Assembler::notZero, *slow_path->entry());
3255 }
3256 }
3257
3258
3259 // This code replaces a call to arraycopy; no exception may
3260 // be thrown in this code, they must be thrown in the System.arraycopy
3261 // activation frame; we could save some checks if this would not be the case
3262 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3263 ciArrayKlass* default_type = op->expected_type();
3264 Register src = op->src()->as_register();
3265 Register dst = op->dst()->as_register();
3266 Register src_pos = op->src_pos()->as_register();
3267 Register dst_pos = op->dst_pos()->as_register();
3268 Register length = op->length()->as_register();
3269 Register tmp = op->tmp()->as_register();
3270 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3271
3272 CodeStub* stub = op->stub();
3273 int flags = op->flags();
3274 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
3275 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
3276
3277 if (flags & LIR_OpArrayCopy::always_slow_path) {
3278 __ jmp(*stub->entry());
3279 __ bind(*stub->continuation());
3280 return;
3281 }
3282
3283 // if we don't know anything, just go through the generic arraycopy
3284 if (default_type == nullptr) {
3285 // save outgoing arguments on stack in case call to System.arraycopy is needed
3286 // HACK ALERT. This code used to push the parameters in a hardwired fashion
3287 // for interpreter calling conventions. Now we have to do it in new style conventions.
3288 // For the moment until C1 gets the new register allocator I just force all the
3289 // args to the right place (except the register args) and then on the back side
3290 // reload the register args properly if we go slow path. Yuck
3291
3292 // These are proper for the calling convention
3293 store_parameter(length, 2);
3294 store_parameter(dst_pos, 1);
3295 store_parameter(dst, 0);
3296
3297 // these are just temporary placements until we need to reload
3298 store_parameter(src_pos, 3);
3299 store_parameter(src, 4);
3300 NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3301
3302 address copyfunc_addr = StubRoutines::generic_arraycopy();
3356 __ mov(tmp, rax);
3357 __ xorl(tmp, -1);
3358
3359 // Reload values from the stack so they are where the stub
3360 // expects them.
3361 __ movptr (dst, Address(rsp, 0*BytesPerWord));
3362 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
3363 __ movptr (length, Address(rsp, 2*BytesPerWord));
3364 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
3365 __ movptr (src, Address(rsp, 4*BytesPerWord));
3366
3367 __ subl(length, tmp);
3368 __ addl(src_pos, tmp);
3369 __ addl(dst_pos, tmp);
3370 __ jmp(*stub->entry());
3371
3372 __ bind(*stub->continuation());
3373 return;
3374 }
3375
3376 // Handle inline type arrays
3377 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
3378 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
3379 }
3380 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
3381 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
3382 }
3383
3384 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3385
3386 int elem_size = type2aelembytes(basic_type);
3387 Address::ScaleFactor scale;
3388
3389 switch (elem_size) {
3390 case 1 :
3391 scale = Address::times_1;
3392 break;
3393 case 2 :
3394 scale = Address::times_2;
3395 break;
3396 case 4 :
3397 scale = Address::times_4;
3398 break;
3399 case 8 :
3400 scale = Address::times_8;
3401 break;
3402 default:
3403 scale = Address::no_scale;
3965 __ jccb(Assembler::zero, next);
3966 #endif
3967 // first time here. Set profile type.
3968 __ movptr(mdo_addr, tmp);
3969 } else {
3970 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3971 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3972
3973 __ movptr(tmp, mdo_addr);
3974 __ testptr(tmp, TypeEntries::type_unknown);
3975 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3976
3977 __ orptr(mdo_addr, TypeEntries::type_unknown);
3978 }
3979 }
3980
3981 __ bind(next);
3982 }
3983 }
3984
3985 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3986 Register obj = op->obj()->as_register();
3987 Register tmp = op->tmp()->as_pointer_register();
3988 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3989 bool not_null = op->not_null();
3990 int flag = op->flag();
3991
3992 Label not_inline_type;
3993 if (!not_null) {
3994 __ testptr(obj, obj);
3995 __ jccb(Assembler::zero, not_inline_type);
3996 }
3997
3998 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3999
4000 __ orb(mdo_addr, flag);
4001
4002 __ bind(not_inline_type);
4003 }
4004
4005 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
4006 Unimplemented();
4007 }
4008
4009
4010 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
4011 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
4012 }
4013
4014
4015 void LIR_Assembler::align_backward_branch_target() {
4016 __ align(BytesPerWord);
4017 }
4018
4019
4020 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
4021 if (left->is_single_cpu()) {
4022 __ negl(left->as_register());
4023 move_regs(left->as_register(), dest->as_register());
4024
4248 }
4249
4250 void LIR_Assembler::membar_storeload() {
4251 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
4252 }
4253
4254 void LIR_Assembler::on_spin_wait() {
4255 __ pause ();
4256 }
4257
4258 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
4259 assert(result_reg->is_register(), "check");
4260 #ifdef _LP64
4261 // __ get_thread(result_reg->as_register_lo());
4262 __ mov(result_reg->as_register(), r15_thread);
4263 #else
4264 __ get_thread(result_reg->as_register());
4265 #endif // _LP64
4266 }
4267
4268 void LIR_Assembler::check_orig_pc() {
4269 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
4270 }
4271
4272 void LIR_Assembler::peephole(LIR_List*) {
4273 // do nothing for now
4274 }
4275
4276 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
4277 assert(data == dest, "xchg/xadd uses only 2 operands");
4278
4279 if (data->type() == T_INT) {
4280 if (code == lir_xadd) {
4281 __ lock();
4282 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
4283 } else {
4284 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
4285 }
4286 } else if (data->is_oop()) {
4287 assert (code == lir_xchg, "xadd for oops");
4288 Register obj = data->as_register();
4289 #ifdef _LP64
4290 if (UseCompressedOops) {
|