15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/objArrayKlass.hpp"
41 #include "runtime/frame.inline.hpp"
42 #include "runtime/safepointMechanism.hpp"
43 #include "runtime/sharedRuntime.hpp"
44 #include "runtime/stubRoutines.hpp"
45 #include "utilities/powerOfTwo.hpp"
46 #include "vmreg_x86.inline.hpp"
47
48
49 // These masks are used to provide 128-bit aligned bitmasks to the XMM
50 // instructions, to allow sign-masking or sign-bit flipping. They allow
51 // fast versions of NegF/NegD and AbsF/AbsD.
52
53 // Note: 'double' and 'long long' have 32-bits alignment on x86.
54 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
55 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
56 // of 128-bits operands for SSE instructions.
57 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
58 // Store the value to a 128-bits operand.
59 operand[0] = lo;
445 __ bind(*stub->continuation());
446 }
447
448 if (compilation()->env()->dtrace_method_probes()) {
449 #ifdef _LP64
450 __ mov(rdi, r15_thread);
451 __ mov_metadata(rsi, method()->constant_encoding());
452 #else
453 __ get_thread(rax);
454 __ movptr(Address(rsp, 0), rax);
455 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
456 #endif
457 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
458 }
459
460 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
461 __ mov(rax, rbx); // Restore the exception
462 }
463
464 // remove the activation and dispatch to the unwind handler
465 __ remove_frame(initial_frame_size_in_bytes());
466 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
467
468 // Emit the slow path assembly
469 if (stub != nullptr) {
470 stub->emit_code(this);
471 }
472
473 return offset;
474 }
475
476
477 int LIR_Assembler::emit_deopt_handler() {
478 // generate code for exception handler
479 address handler_base = __ start_a_stub(deopt_handler_size());
480 if (handler_base == nullptr) {
481 // not enough space left for the handler
482 bailout("deopt handler overflow");
483 return -1;
484 }
485
486 int offset = code_offset();
487 InternalAddress here(__ pc());
488
489 __ pushptr(here.addr(), rscratch1);
490 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
491 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
492 __ end_a_stub();
493
494 return offset;
495 }
496
497 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
498 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
499 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
500 assert(result->fpu() == 0, "result must already be on TOS");
501 }
502
503 // Pop the stack before the safepoint code
504 __ remove_frame(initial_frame_size_in_bytes());
505
506 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
507 __ reserved_stack_check();
508 }
509
510 // Note: we do not need to round double result; float result has the right precision
511 // the poll sets the condition code, but no data registers
512
513 #ifdef _LP64
514 const Register thread = r15_thread;
515 #else
516 const Register thread = rbx;
517 __ get_thread(thread);
518 #endif
519 code_stub->set_safepoint_offset(__ offset());
520 __ relocate(relocInfo::poll_return_type);
521 __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
522 __ ret(0);
523 }
524
525
526 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
527 guarantee(info != nullptr, "Shouldn't be null");
528 int offset = __ offset();
529 #ifdef _LP64
530 const Register poll_addr = rscratch1;
531 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
532 #else
533 assert(tmp->is_cpu_register(), "needed");
534 const Register poll_addr = tmp->as_register();
535 __ get_thread(poll_addr);
536 __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
537 #endif
538 add_debug_info_for_branch(info);
539 __ relocate(relocInfo::poll_type);
540 address pre_pc = __ pc();
541 __ testl(rax, Address(poll_addr, 0));
542 address post_pc = __ pc();
543 guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
544 return offset;
545 }
1578 // init_state needs acquire, but x86 is TSO, and so we are already good.
1579 __ cmpb(Address(op->klass()->as_register(),
1580 InstanceKlass::init_state_offset()),
1581 InstanceKlass::fully_initialized);
1582 __ jcc(Assembler::notEqual, *op->stub()->entry());
1583 }
1584 __ allocate_object(op->obj()->as_register(),
1585 op->tmp1()->as_register(),
1586 op->tmp2()->as_register(),
1587 op->header_size(),
1588 op->object_size(),
1589 op->klass()->as_register(),
1590 *op->stub()->entry());
1591 __ bind(*op->stub()->continuation());
1592 }
1593
1594 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1595 Register len = op->len()->as_register();
1596 LP64_ONLY( __ movslq(len, len); )
1597
1598 if (UseSlowPath ||
1599 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1600 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1601 __ jmp(*op->stub()->entry());
1602 } else {
1603 Register tmp1 = op->tmp1()->as_register();
1604 Register tmp2 = op->tmp2()->as_register();
1605 Register tmp3 = op->tmp3()->as_register();
1606 if (len == tmp1) {
1607 tmp1 = tmp3;
1608 } else if (len == tmp2) {
1609 tmp2 = tmp3;
1610 } else if (len == tmp3) {
1611 // everything is ok
1612 } else {
1613 __ mov(tmp3, len);
1614 }
1615 __ allocate_array(op->obj()->as_register(),
1616 len,
1617 tmp1,
1618 tmp2,
1677 assert(data != nullptr, "need data for type check");
1678 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1679 }
1680 Label* success_target = success;
1681 Label* failure_target = failure;
1682
1683 if (obj == k_RInfo) {
1684 k_RInfo = dst;
1685 } else if (obj == klass_RInfo) {
1686 klass_RInfo = dst;
1687 }
1688 if (k->is_loaded() && !UseCompressedClassPointers) {
1689 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1690 } else {
1691 Rtmp1 = op->tmp3()->as_register();
1692 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1693 }
1694
1695 assert_different_registers(obj, k_RInfo, klass_RInfo);
1696
1697 __ testptr(obj, obj);
1698 if (op->should_profile()) {
1699 Label not_null;
1700 Register mdo = klass_RInfo;
1701 __ mov_metadata(mdo, md->constant_encoding());
1702 __ jccb(Assembler::notEqual, not_null);
1703 // Object is null; update MDO and exit
1704 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1705 int header_bits = BitData::null_seen_byte_constant();
1706 __ orb(data_addr, header_bits);
1707 __ jmp(*obj_is_null);
1708 __ bind(not_null);
1709
1710 Label update_done;
1711 Register recv = k_RInfo;
1712 __ load_klass(recv, obj, tmp_load_klass);
1713 type_profile_helper(mdo, md, data, recv, &update_done);
1714
1715 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1716 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1717
1718 __ bind(update_done);
1719 } else {
1720 __ jcc(Assembler::equal, *obj_is_null);
1721 }
1722
1723 if (!k->is_loaded()) {
1724 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1725 } else {
1726 #ifdef _LP64
1727 __ mov_metadata(k_RInfo, k->constant_encoding());
1728 #endif // _LP64
1729 }
1730 __ verify_oop(obj);
1731
1732 if (op->fast_check()) {
1733 // get object class
1734 // not a safepoint as obj null check happens earlier
1735 #ifdef _LP64
1736 if (UseCompressedClassPointers) {
1737 __ load_klass(Rtmp1, obj, tmp_load_klass);
1738 __ cmpptr(k_RInfo, Rtmp1);
1739 } else {
1740 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1892 __ mov(dst, obj);
1893 }
1894 } else
1895 if (code == lir_instanceof) {
1896 Register obj = op->object()->as_register();
1897 Register dst = op->result_opr()->as_register();
1898 Label success, failure, done;
1899 emit_typecheck_helper(op, &success, &failure, &failure);
1900 __ bind(failure);
1901 __ xorptr(dst, dst);
1902 __ jmpb(done);
1903 __ bind(success);
1904 __ movptr(dst, 1);
1905 __ bind(done);
1906 } else {
1907 ShouldNotReachHere();
1908 }
1909
1910 }
1911
1912
1913 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1914 if (LP64_ONLY(false &&) op->code() == lir_cas_long) {
1915 assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1916 assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1917 assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1918 assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1919 Register addr = op->addr()->as_register();
1920 __ lock();
1921 NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1922
1923 } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1924 NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1925 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1926 Register newval = op->new_value()->as_register();
1927 Register cmpval = op->cmp_value()->as_register();
1928 assert(cmpval == rax, "wrong register");
1929 assert(newval != noreg, "new val must be register");
1930 assert(cmpval != newval, "cmp and new values must be in different registers");
1931 assert(cmpval != addr, "cmp and addr must be in different registers");
1952 __ cmpxchgl(newval, Address(addr, 0));
1953 }
1954 #ifdef _LP64
1955 } else if (op->code() == lir_cas_long) {
1956 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1957 Register newval = op->new_value()->as_register_lo();
1958 Register cmpval = op->cmp_value()->as_register_lo();
1959 assert(cmpval == rax, "wrong register");
1960 assert(newval != noreg, "new val must be register");
1961 assert(cmpval != newval, "cmp and new values must be in different registers");
1962 assert(cmpval != addr, "cmp and addr must be in different registers");
1963 assert(newval != addr, "new value and addr must be in different registers");
1964 __ lock();
1965 __ cmpxchgq(newval, Address(addr, 0));
1966 #endif // _LP64
1967 } else {
1968 Unimplemented();
1969 }
1970 }
1971
1972 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1973 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1974 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1975
1976 Assembler::Condition acond, ncond;
1977 switch (condition) {
1978 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1979 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1980 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1981 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1982 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1983 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1984 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1985 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1986 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1987 ShouldNotReachHere();
1988 }
1989
1990 if (opr1->is_cpu_register()) {
1991 reg2reg(opr1, result);
2829 int offset = __ offset();
2830 switch (code) {
2831 case lir_static_call:
2832 case lir_optvirtual_call:
2833 case lir_dynamic_call:
2834 offset += NativeCall::displacement_offset;
2835 break;
2836 case lir_icvirtual_call:
2837 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2838 break;
2839 default: ShouldNotReachHere();
2840 }
2841 __ align(BytesPerWord, offset);
2842 }
2843
2844
2845 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2846 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2847 "must be aligned");
2848 __ call(AddressLiteral(op->addr(), rtype));
2849 add_call_info(code_offset(), op->info());
2850 __ post_call_nop();
2851 }
2852
2853
2854 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2855 __ ic_call(op->addr());
2856 add_call_info(code_offset(), op->info());
2857 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2858 "must be aligned");
2859 __ post_call_nop();
2860 }
2861
2862
2863 void LIR_Assembler::emit_static_call_stub() {
2864 address call_pc = __ pc();
2865 address stub = __ start_a_stub(call_stub_size());
2866 if (stub == nullptr) {
2867 bailout("static call stub overflow");
2868 return;
2869 }
2870
2871 int start = __ offset();
2872
2873 // make sure that the displacement word of the call ends up word aligned
2874 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2875 __ relocate(static_stub_Relocation::spec(call_pc));
2876 __ mov_metadata(rbx, (Metadata*)nullptr);
3017 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3018 }
3019
3020
3021 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
3022 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3023 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3024 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3025 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
3026 }
3027
3028
3029 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
3030 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3031 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3032 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3033 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
3034 }
3035
3036
3037 // This code replaces a call to arraycopy; no exception may
3038 // be thrown in this code, they must be thrown in the System.arraycopy
3039 // activation frame; we could save some checks if this would not be the case
3040 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3041 ciArrayKlass* default_type = op->expected_type();
3042 Register src = op->src()->as_register();
3043 Register dst = op->dst()->as_register();
3044 Register src_pos = op->src_pos()->as_register();
3045 Register dst_pos = op->dst_pos()->as_register();
3046 Register length = op->length()->as_register();
3047 Register tmp = op->tmp()->as_register();
3048 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3049
3050 CodeStub* stub = op->stub();
3051 int flags = op->flags();
3052 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
3053 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
3054
3055 // if we don't know anything, just go through the generic arraycopy
3056 if (default_type == nullptr) {
3057 // save outgoing arguments on stack in case call to System.arraycopy is needed
3058 // HACK ALERT. This code used to push the parameters in a hardwired fashion
3059 // for interpreter calling conventions. Now we have to do it in new style conventions.
3060 // For the moment until C1 gets the new register allocator I just force all the
3061 // args to the right place (except the register args) and then on the back side
3062 // reload the register args properly if we go slow path. Yuck
3063
3064 // These are proper for the calling convention
3065 store_parameter(length, 2);
3066 store_parameter(dst_pos, 1);
3067 store_parameter(dst, 0);
3068
3069 // these are just temporary placements until we need to reload
3070 store_parameter(src_pos, 3);
3071 store_parameter(src, 4);
3072 NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3073
3074 address copyfunc_addr = StubRoutines::generic_arraycopy();
3128 __ mov(tmp, rax);
3129 __ xorl(tmp, -1);
3130
3131 // Reload values from the stack so they are where the stub
3132 // expects them.
3133 __ movptr (dst, Address(rsp, 0*BytesPerWord));
3134 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
3135 __ movptr (length, Address(rsp, 2*BytesPerWord));
3136 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
3137 __ movptr (src, Address(rsp, 4*BytesPerWord));
3138
3139 __ subl(length, tmp);
3140 __ addl(src_pos, tmp);
3141 __ addl(dst_pos, tmp);
3142 __ jmp(*stub->entry());
3143
3144 __ bind(*stub->continuation());
3145 return;
3146 }
3147
3148 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3149
3150 int elem_size = type2aelembytes(basic_type);
3151 Address::ScaleFactor scale;
3152
3153 switch (elem_size) {
3154 case 1 :
3155 scale = Address::times_1;
3156 break;
3157 case 2 :
3158 scale = Address::times_2;
3159 break;
3160 case 4 :
3161 scale = Address::times_4;
3162 break;
3163 case 8 :
3164 scale = Address::times_8;
3165 break;
3166 default:
3167 scale = Address::no_scale;
3757 // first time here. Set profile type.
3758 __ movptr(mdo_addr, tmp);
3759 #ifdef ASSERT
3760 __ andptr(tmp, TypeEntries::type_klass_mask);
3761 __ verify_klass_ptr(tmp);
3762 #endif
3763 } else {
3764 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3765 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3766
3767 __ testptr(mdo_addr, TypeEntries::type_unknown);
3768 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3769
3770 __ orptr(mdo_addr, TypeEntries::type_unknown);
3771 }
3772 }
3773 }
3774 __ bind(next);
3775 }
3776
3777 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3778 Unimplemented();
3779 }
3780
3781
3782 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3783 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3784 }
3785
3786
3787 void LIR_Assembler::align_backward_branch_target() {
3788 __ align(BytesPerWord);
3789 }
3790
3791
3792 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3793 if (left->is_single_cpu()) {
3794 __ negl(left->as_register());
3795 move_regs(left->as_register(), dest->as_register());
3796
4020 }
4021
4022 void LIR_Assembler::membar_storeload() {
4023 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
4024 }
4025
4026 void LIR_Assembler::on_spin_wait() {
4027 __ pause ();
4028 }
4029
4030 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
4031 assert(result_reg->is_register(), "check");
4032 #ifdef _LP64
4033 // __ get_thread(result_reg->as_register_lo());
4034 __ mov(result_reg->as_register(), r15_thread);
4035 #else
4036 __ get_thread(result_reg->as_register());
4037 #endif // _LP64
4038 }
4039
4040
4041 void LIR_Assembler::peephole(LIR_List*) {
4042 // do nothing for now
4043 }
4044
4045 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
4046 assert(data == dest, "xchg/xadd uses only 2 operands");
4047
4048 if (data->type() == T_INT) {
4049 if (code == lir_xadd) {
4050 __ lock();
4051 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
4052 } else {
4053 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
4054 }
4055 } else if (data->is_oop()) {
4056 assert (code == lir_xchg, "xadd for oops");
4057 Register obj = data->as_register();
4058 #ifdef _LP64
4059 if (UseCompressedOops) {
|
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "precompiled.hpp"
26 #include "asm/macroAssembler.hpp"
27 #include "asm/macroAssembler.inline.hpp"
28 #include "c1/c1_CodeStubs.hpp"
29 #include "c1/c1_Compilation.hpp"
30 #include "c1/c1_LIRAssembler.hpp"
31 #include "c1/c1_MacroAssembler.hpp"
32 #include "c1/c1_Runtime1.hpp"
33 #include "c1/c1_ValueStack.hpp"
34 #include "ci/ciArrayKlass.hpp"
35 #include "ci/ciInlineKlass.hpp"
36 #include "ci/ciInstance.hpp"
37 #include "compiler/oopMap.hpp"
38 #include "gc/shared/collectedHeap.hpp"
39 #include "gc/shared/gc_globals.hpp"
40 #include "nativeInst_x86.hpp"
41 #include "oops/oop.inline.hpp"
42 #include "oops/objArrayKlass.hpp"
43 #include "runtime/frame.inline.hpp"
44 #include "runtime/safepointMechanism.hpp"
45 #include "runtime/sharedRuntime.hpp"
46 #include "runtime/stubRoutines.hpp"
47 #include "utilities/powerOfTwo.hpp"
48 #include "vmreg_x86.inline.hpp"
49
50
51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
52 // instructions, to allow sign-masking or sign-bit flipping. They allow
53 // fast versions of NegF/NegD and AbsF/AbsD.
54
55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
57 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
58 // of 128-bits operands for SSE instructions.
59 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
60 // Store the value to a 128-bits operand.
61 operand[0] = lo;
447 __ bind(*stub->continuation());
448 }
449
450 if (compilation()->env()->dtrace_method_probes()) {
451 #ifdef _LP64
452 __ mov(rdi, r15_thread);
453 __ mov_metadata(rsi, method()->constant_encoding());
454 #else
455 __ get_thread(rax);
456 __ movptr(Address(rsp, 0), rax);
457 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
458 #endif
459 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
460 }
461
462 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
463 __ mov(rax, rbx); // Restore the exception
464 }
465
466 // remove the activation and dispatch to the unwind handler
467 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
468 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
469
470 // Emit the slow path assembly
471 if (stub != nullptr) {
472 stub->emit_code(this);
473 }
474
475 return offset;
476 }
477
478
479 int LIR_Assembler::emit_deopt_handler() {
480 // generate code for exception handler
481 address handler_base = __ start_a_stub(deopt_handler_size());
482 if (handler_base == nullptr) {
483 // not enough space left for the handler
484 bailout("deopt handler overflow");
485 return -1;
486 }
487
488 int offset = code_offset();
489 InternalAddress here(__ pc());
490
491 __ pushptr(here.addr(), rscratch1);
492 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
493 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
494 __ end_a_stub();
495
496 return offset;
497 }
498
499 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
500 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
501 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
502 assert(result->fpu() == 0, "result must already be on TOS");
503 }
504 if (InlineTypeReturnedAsFields) {
505 #ifndef _LP64
506 Unimplemented();
507 #endif
508 // Check if we are returning an non-null inline type and load its fields into registers
509 ciType* return_type = compilation()->method()->return_type();
510 if (return_type->is_inlinetype()) {
511 ciInlineKlass* vk = return_type->as_inline_klass();
512 if (vk->can_be_returned_as_fields()) {
513 address unpack_handler = vk->unpack_handler();
514 assert(unpack_handler != nullptr, "must be");
515 __ call(RuntimeAddress(unpack_handler));
516 }
517 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
518 Label skip;
519 __ test_oop_is_not_inline_type(rax, rscratch1, skip);
520
521 // Load fields from a buffered value with an inline class specific handler
522 __ load_klass(rdi, rax, rscratch1);
523 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
524 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
525 // Unpack handler can be null if inline type is not scalarizable in returns
526 __ testptr(rdi, rdi);
527 __ jcc(Assembler::zero, skip);
528 __ call(rdi);
529
530 __ bind(skip);
531 }
532 // At this point, rax points to the value object (for interpreter or C1 caller).
533 // The fields of the object are copied into registers (for C2 caller).
534 }
535
536 // Pop the stack before the safepoint code
537 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
538
539 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
540 __ reserved_stack_check();
541 }
542
543 // Note: we do not need to round double result; float result has the right precision
544 // the poll sets the condition code, but no data registers
545
546 #ifdef _LP64
547 const Register thread = r15_thread;
548 #else
549 const Register thread = rbx;
550 __ get_thread(thread);
551 #endif
552 code_stub->set_safepoint_offset(__ offset());
553 __ relocate(relocInfo::poll_return_type);
554 __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
555 __ ret(0);
556 }
557
558
559 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
560 return (__ store_inline_type_fields_to_buf(vk, false));
561 }
562
563 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
564 guarantee(info != nullptr, "Shouldn't be null");
565 int offset = __ offset();
566 #ifdef _LP64
567 const Register poll_addr = rscratch1;
568 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
569 #else
570 assert(tmp->is_cpu_register(), "needed");
571 const Register poll_addr = tmp->as_register();
572 __ get_thread(poll_addr);
573 __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
574 #endif
575 add_debug_info_for_branch(info);
576 __ relocate(relocInfo::poll_type);
577 address pre_pc = __ pc();
578 __ testl(rax, Address(poll_addr, 0));
579 address post_pc = __ pc();
580 guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
581 return offset;
582 }
1615 // init_state needs acquire, but x86 is TSO, and so we are already good.
1616 __ cmpb(Address(op->klass()->as_register(),
1617 InstanceKlass::init_state_offset()),
1618 InstanceKlass::fully_initialized);
1619 __ jcc(Assembler::notEqual, *op->stub()->entry());
1620 }
1621 __ allocate_object(op->obj()->as_register(),
1622 op->tmp1()->as_register(),
1623 op->tmp2()->as_register(),
1624 op->header_size(),
1625 op->object_size(),
1626 op->klass()->as_register(),
1627 *op->stub()->entry());
1628 __ bind(*op->stub()->continuation());
1629 }
1630
1631 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1632 Register len = op->len()->as_register();
1633 LP64_ONLY( __ movslq(len, len); )
1634
1635 if (UseSlowPath || op->is_null_free() ||
1636 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1637 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1638 __ jmp(*op->stub()->entry());
1639 } else {
1640 Register tmp1 = op->tmp1()->as_register();
1641 Register tmp2 = op->tmp2()->as_register();
1642 Register tmp3 = op->tmp3()->as_register();
1643 if (len == tmp1) {
1644 tmp1 = tmp3;
1645 } else if (len == tmp2) {
1646 tmp2 = tmp3;
1647 } else if (len == tmp3) {
1648 // everything is ok
1649 } else {
1650 __ mov(tmp3, len);
1651 }
1652 __ allocate_array(op->obj()->as_register(),
1653 len,
1654 tmp1,
1655 tmp2,
1714 assert(data != nullptr, "need data for type check");
1715 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1716 }
1717 Label* success_target = success;
1718 Label* failure_target = failure;
1719
1720 if (obj == k_RInfo) {
1721 k_RInfo = dst;
1722 } else if (obj == klass_RInfo) {
1723 klass_RInfo = dst;
1724 }
1725 if (k->is_loaded() && !UseCompressedClassPointers) {
1726 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1727 } else {
1728 Rtmp1 = op->tmp3()->as_register();
1729 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1730 }
1731
1732 assert_different_registers(obj, k_RInfo, klass_RInfo);
1733
1734 if (op->need_null_check()) {
1735 __ testptr(obj, obj);
1736 if (op->should_profile()) {
1737 Label not_null;
1738 Register mdo = klass_RInfo;
1739 __ mov_metadata(mdo, md->constant_encoding());
1740 __ jccb(Assembler::notEqual, not_null);
1741 // Object is null; update MDO and exit
1742 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1743 int header_bits = BitData::null_seen_byte_constant();
1744 __ orb(data_addr, header_bits);
1745 __ jmp(*obj_is_null);
1746 __ bind(not_null);
1747
1748 Label update_done;
1749 Register recv = k_RInfo;
1750 __ load_klass(recv, obj, tmp_load_klass);
1751 type_profile_helper(mdo, md, data, recv, &update_done);
1752
1753 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1754 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1755
1756 __ bind(update_done);
1757 } else {
1758 __ jcc(Assembler::equal, *obj_is_null);
1759 }
1760 }
1761
1762 if (!k->is_loaded()) {
1763 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1764 } else {
1765 #ifdef _LP64
1766 __ mov_metadata(k_RInfo, k->constant_encoding());
1767 #endif // _LP64
1768 }
1769 __ verify_oop(obj);
1770
1771 if (op->fast_check()) {
1772 // get object class
1773 // not a safepoint as obj null check happens earlier
1774 #ifdef _LP64
1775 if (UseCompressedClassPointers) {
1776 __ load_klass(Rtmp1, obj, tmp_load_klass);
1777 __ cmpptr(k_RInfo, Rtmp1);
1778 } else {
1779 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1931 __ mov(dst, obj);
1932 }
1933 } else
1934 if (code == lir_instanceof) {
1935 Register obj = op->object()->as_register();
1936 Register dst = op->result_opr()->as_register();
1937 Label success, failure, done;
1938 emit_typecheck_helper(op, &success, &failure, &failure);
1939 __ bind(failure);
1940 __ xorptr(dst, dst);
1941 __ jmpb(done);
1942 __ bind(success);
1943 __ movptr(dst, 1);
1944 __ bind(done);
1945 } else {
1946 ShouldNotReachHere();
1947 }
1948
1949 }
1950
1951 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1952 // We are loading/storing from/to an array that *may* be a flat array (the
1953 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1954 // If this array is a flat array, take the slow path.
1955 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1956 if (!op->value()->is_illegal()) {
1957 // The array is not a flat array, but it might be null-free. If we are storing
1958 // a null into a null-free array, take the slow path (which will throw NPE).
1959 Label skip;
1960 __ cmpptr(op->value()->as_register(), NULL_WORD);
1961 __ jcc(Assembler::notEqual, skip);
1962 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1963 __ bind(skip);
1964 }
1965 }
1966
1967 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1968 // We are storing into an array that *may* be null-free (the declared type is
1969 // Object[], abstract[], interface[] or VT.ref[]).
1970 Label test_mark_word;
1971 Register tmp = op->tmp()->as_register();
1972 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1973 __ testl(tmp, markWord::unlocked_value);
1974 __ jccb(Assembler::notZero, test_mark_word);
1975 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1976 __ bind(test_mark_word);
1977 __ testl(tmp, markWord::null_free_array_bit_in_place);
1978 }
1979
1980 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1981 Label L_oops_equal;
1982 Label L_oops_not_equal;
1983 Label L_end;
1984
1985 Register left = op->left()->as_register();
1986 Register right = op->right()->as_register();
1987
1988 __ cmpptr(left, right);
1989 __ jcc(Assembler::equal, L_oops_equal);
1990
1991 // (1) Null check -- if one of the operands is null, the other must not be null (because
1992 // the two references are not equal), so they are not substitutable,
1993 // FIXME: do null check only if the operand is nullable
1994 __ testptr(left, right);
1995 __ jcc(Assembler::zero, L_oops_not_equal);
1996
1997 ciKlass* left_klass = op->left_klass();
1998 ciKlass* right_klass = op->right_klass();
1999
2000 // (2) Inline type check -- if either of the operands is not a inline type,
2001 // they are not substitutable. We do this only if we are not sure that the
2002 // operands are inline type
2003 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
2004 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
2005 Register tmp1 = op->tmp1()->as_register();
2006 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
2007 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
2008 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
2009 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
2010 __ jcc(Assembler::notEqual, L_oops_not_equal);
2011 }
2012
2013 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
2014 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
2015 // No need to load klass -- the operands are statically known to be the same inline klass.
2016 __ jmp(*op->stub()->entry());
2017 } else {
2018 Register left_klass_op = op->left_klass_op()->as_register();
2019 Register right_klass_op = op->right_klass_op()->as_register();
2020
2021 if (UseCompressedClassPointers) {
2022 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
2023 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2024 __ cmpl(left_klass_op, right_klass_op);
2025 } else {
2026 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
2027 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2028 __ cmpptr(left_klass_op, right_klass_op);
2029 }
2030
2031 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
2032 // fall through to L_oops_not_equal
2033 }
2034
2035 __ bind(L_oops_not_equal);
2036 move(op->not_equal_result(), op->result_opr());
2037 __ jmp(L_end);
2038
2039 __ bind(L_oops_equal);
2040 move(op->equal_result(), op->result_opr());
2041 __ jmp(L_end);
2042
2043 // We've returned from the stub. RAX contains 0x0 IFF the two
2044 // operands are not substitutable. (Don't compare against 0x1 in case the
2045 // C compiler is naughty)
2046 __ bind(*op->stub()->continuation());
2047 __ cmpl(rax, 0);
2048 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
2049 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
2050 // fall-through
2051 __ bind(L_end);
2052 }
2053
2054 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2055 if (LP64_ONLY(false &&) op->code() == lir_cas_long) {
2056 assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
2057 assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
2058 assert(op->new_value()->as_register_lo() == rbx, "wrong register");
2059 assert(op->new_value()->as_register_hi() == rcx, "wrong register");
2060 Register addr = op->addr()->as_register();
2061 __ lock();
2062 NOT_LP64(__ cmpxchg8(Address(addr, 0)));
2063
2064 } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
2065 NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
2066 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2067 Register newval = op->new_value()->as_register();
2068 Register cmpval = op->cmp_value()->as_register();
2069 assert(cmpval == rax, "wrong register");
2070 assert(newval != noreg, "new val must be register");
2071 assert(cmpval != newval, "cmp and new values must be in different registers");
2072 assert(cmpval != addr, "cmp and addr must be in different registers");
2093 __ cmpxchgl(newval, Address(addr, 0));
2094 }
2095 #ifdef _LP64
2096 } else if (op->code() == lir_cas_long) {
2097 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2098 Register newval = op->new_value()->as_register_lo();
2099 Register cmpval = op->cmp_value()->as_register_lo();
2100 assert(cmpval == rax, "wrong register");
2101 assert(newval != noreg, "new val must be register");
2102 assert(cmpval != newval, "cmp and new values must be in different registers");
2103 assert(cmpval != addr, "cmp and addr must be in different registers");
2104 assert(newval != addr, "new value and addr must be in different registers");
2105 __ lock();
2106 __ cmpxchgq(newval, Address(addr, 0));
2107 #endif // _LP64
2108 } else {
2109 Unimplemented();
2110 }
2111 }
2112
2113 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
2114 assert(dst->is_cpu_register(), "must be");
2115 assert(dst->type() == src->type(), "must be");
2116
2117 if (src->is_cpu_register()) {
2118 reg2reg(src, dst);
2119 } else if (src->is_stack()) {
2120 stack2reg(src, dst, dst->type());
2121 } else if (src->is_constant()) {
2122 const2reg(src, dst, lir_patch_none, nullptr);
2123 } else {
2124 ShouldNotReachHere();
2125 }
2126 }
2127
2128 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
2129 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
2130 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
2131
2132 Assembler::Condition acond, ncond;
2133 switch (condition) {
2134 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
2135 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
2136 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
2137 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
2138 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
2139 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
2140 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
2141 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
2142 default: acond = Assembler::equal; ncond = Assembler::notEqual;
2143 ShouldNotReachHere();
2144 }
2145
2146 if (opr1->is_cpu_register()) {
2147 reg2reg(opr1, result);
2985 int offset = __ offset();
2986 switch (code) {
2987 case lir_static_call:
2988 case lir_optvirtual_call:
2989 case lir_dynamic_call:
2990 offset += NativeCall::displacement_offset;
2991 break;
2992 case lir_icvirtual_call:
2993 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2994 break;
2995 default: ShouldNotReachHere();
2996 }
2997 __ align(BytesPerWord, offset);
2998 }
2999
3000
3001 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
3002 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
3003 "must be aligned");
3004 __ call(AddressLiteral(op->addr(), rtype));
3005 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
3006 __ post_call_nop();
3007 }
3008
3009
3010 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
3011 __ ic_call(op->addr());
3012 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
3013 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
3014 "must be aligned");
3015 __ post_call_nop();
3016 }
3017
3018
3019 void LIR_Assembler::emit_static_call_stub() {
3020 address call_pc = __ pc();
3021 address stub = __ start_a_stub(call_stub_size());
3022 if (stub == nullptr) {
3023 bailout("static call stub overflow");
3024 return;
3025 }
3026
3027 int start = __ offset();
3028
3029 // make sure that the displacement word of the call ends up word aligned
3030 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
3031 __ relocate(static_stub_Relocation::spec(call_pc));
3032 __ mov_metadata(rbx, (Metadata*)nullptr);
3173 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3174 }
3175
3176
3177 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
3178 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3179 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3180 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3181 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
3182 }
3183
3184
3185 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
3186 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3187 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3188 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3189 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
3190 }
3191
3192
3193 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
3194 if (null_check) {
3195 __ testptr(obj, obj);
3196 __ jcc(Assembler::zero, *slow_path->entry());
3197 }
3198 if (is_dest) {
3199 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
3200 } else {
3201 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
3202 }
3203 }
3204
3205
3206 // This code replaces a call to arraycopy; no exception may
3207 // be thrown in this code, they must be thrown in the System.arraycopy
3208 // activation frame; we could save some checks if this would not be the case
3209 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3210 ciArrayKlass* default_type = op->expected_type();
3211 Register src = op->src()->as_register();
3212 Register dst = op->dst()->as_register();
3213 Register src_pos = op->src_pos()->as_register();
3214 Register dst_pos = op->dst_pos()->as_register();
3215 Register length = op->length()->as_register();
3216 Register tmp = op->tmp()->as_register();
3217 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3218
3219 CodeStub* stub = op->stub();
3220 int flags = op->flags();
3221 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
3222 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
3223
3224 if (flags & LIR_OpArrayCopy::always_slow_path) {
3225 __ jmp(*stub->entry());
3226 __ bind(*stub->continuation());
3227 return;
3228 }
3229
3230 // if we don't know anything, just go through the generic arraycopy
3231 if (default_type == nullptr) {
3232 // save outgoing arguments on stack in case call to System.arraycopy is needed
3233 // HACK ALERT. This code used to push the parameters in a hardwired fashion
3234 // for interpreter calling conventions. Now we have to do it in new style conventions.
3235 // For the moment until C1 gets the new register allocator I just force all the
3236 // args to the right place (except the register args) and then on the back side
3237 // reload the register args properly if we go slow path. Yuck
3238
3239 // These are proper for the calling convention
3240 store_parameter(length, 2);
3241 store_parameter(dst_pos, 1);
3242 store_parameter(dst, 0);
3243
3244 // these are just temporary placements until we need to reload
3245 store_parameter(src_pos, 3);
3246 store_parameter(src, 4);
3247 NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3248
3249 address copyfunc_addr = StubRoutines::generic_arraycopy();
3303 __ mov(tmp, rax);
3304 __ xorl(tmp, -1);
3305
3306 // Reload values from the stack so they are where the stub
3307 // expects them.
3308 __ movptr (dst, Address(rsp, 0*BytesPerWord));
3309 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
3310 __ movptr (length, Address(rsp, 2*BytesPerWord));
3311 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
3312 __ movptr (src, Address(rsp, 4*BytesPerWord));
3313
3314 __ subl(length, tmp);
3315 __ addl(src_pos, tmp);
3316 __ addl(dst_pos, tmp);
3317 __ jmp(*stub->entry());
3318
3319 __ bind(*stub->continuation());
3320 return;
3321 }
3322
3323 // Handle inline type arrays
3324 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
3325 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
3326 }
3327 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
3328 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
3329 }
3330
3331 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3332
3333 int elem_size = type2aelembytes(basic_type);
3334 Address::ScaleFactor scale;
3335
3336 switch (elem_size) {
3337 case 1 :
3338 scale = Address::times_1;
3339 break;
3340 case 2 :
3341 scale = Address::times_2;
3342 break;
3343 case 4 :
3344 scale = Address::times_4;
3345 break;
3346 case 8 :
3347 scale = Address::times_8;
3348 break;
3349 default:
3350 scale = Address::no_scale;
3940 // first time here. Set profile type.
3941 __ movptr(mdo_addr, tmp);
3942 #ifdef ASSERT
3943 __ andptr(tmp, TypeEntries::type_klass_mask);
3944 __ verify_klass_ptr(tmp);
3945 #endif
3946 } else {
3947 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3948 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3949
3950 __ testptr(mdo_addr, TypeEntries::type_unknown);
3951 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3952
3953 __ orptr(mdo_addr, TypeEntries::type_unknown);
3954 }
3955 }
3956 }
3957 __ bind(next);
3958 }
3959
3960 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3961 Register obj = op->obj()->as_register();
3962 Register tmp = op->tmp()->as_pointer_register();
3963 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3964 bool not_null = op->not_null();
3965 int flag = op->flag();
3966
3967 Label not_inline_type;
3968 if (!not_null) {
3969 __ testptr(obj, obj);
3970 __ jccb(Assembler::zero, not_inline_type);
3971 }
3972
3973 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3974
3975 __ orb(mdo_addr, flag);
3976
3977 __ bind(not_inline_type);
3978 }
3979
3980 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3981 Unimplemented();
3982 }
3983
3984
3985 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3986 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3987 }
3988
3989
3990 void LIR_Assembler::align_backward_branch_target() {
3991 __ align(BytesPerWord);
3992 }
3993
3994
3995 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3996 if (left->is_single_cpu()) {
3997 __ negl(left->as_register());
3998 move_regs(left->as_register(), dest->as_register());
3999
4223 }
4224
4225 void LIR_Assembler::membar_storeload() {
4226 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
4227 }
4228
4229 void LIR_Assembler::on_spin_wait() {
4230 __ pause ();
4231 }
4232
4233 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
4234 assert(result_reg->is_register(), "check");
4235 #ifdef _LP64
4236 // __ get_thread(result_reg->as_register_lo());
4237 __ mov(result_reg->as_register(), r15_thread);
4238 #else
4239 __ get_thread(result_reg->as_register());
4240 #endif // _LP64
4241 }
4242
4243 void LIR_Assembler::check_orig_pc() {
4244 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
4245 }
4246
4247 void LIR_Assembler::peephole(LIR_List*) {
4248 // do nothing for now
4249 }
4250
4251 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
4252 assert(data == dest, "xchg/xadd uses only 2 operands");
4253
4254 if (data->type() == T_INT) {
4255 if (code == lir_xadd) {
4256 __ lock();
4257 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
4258 } else {
4259 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
4260 }
4261 } else if (data->is_oop()) {
4262 assert (code == lir_xchg, "xadd for oops");
4263 Register obj = data->as_register();
4264 #ifdef _LP64
4265 if (UseCompressedOops) {
|