< prev index next >

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

Print this page

  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "c1/c1_CodeStubs.hpp"
  29 #include "c1/c1_Compilation.hpp"
  30 #include "c1/c1_LIRAssembler.hpp"
  31 #include "c1/c1_MacroAssembler.hpp"
  32 #include "c1/c1_Runtime1.hpp"
  33 #include "c1/c1_ValueStack.hpp"
  34 #include "ci/ciArrayKlass.hpp"

  35 #include "ci/ciInstance.hpp"
  36 #include "compiler/oopMap.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "gc/shared/gc_globals.hpp"
  39 #include "nativeInst_x86.hpp"

  40 #include "oops/objArrayKlass.hpp"
  41 #include "runtime/frame.inline.hpp"
  42 #include "runtime/safepointMechanism.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "runtime/stubRoutines.hpp"
  45 #include "utilities/powerOfTwo.hpp"
  46 #include "vmreg_x86.inline.hpp"
  47 
  48 
  49 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  50 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  51 // fast versions of NegF/NegD and AbsF/AbsD.
  52 
  53 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  54 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  55   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  56   // of 128-bits operands for SSE instructions.
  57   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  58   // Store the value to a 128-bits operand.
  59   operand[0] = lo;

 462     __ bind(*stub->continuation());
 463   }
 464 
 465   if (compilation()->env()->dtrace_method_probes()) {
 466 #ifdef _LP64
 467     __ mov(rdi, r15_thread);
 468     __ mov_metadata(rsi, method()->constant_encoding());
 469 #else
 470     __ get_thread(rax);
 471     __ movptr(Address(rsp, 0), rax);
 472     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
 473 #endif
 474     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 475   }
 476 
 477   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 478     __ mov(rax, rbx);  // Restore the exception
 479   }
 480 
 481   // remove the activation and dispatch to the unwind handler
 482   __ remove_frame(initial_frame_size_in_bytes());
 483   __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 484 
 485   // Emit the slow path assembly
 486   if (stub != nullptr) {
 487     stub->emit_code(this);
 488   }
 489 
 490   return offset;
 491 }
 492 
 493 
 494 int LIR_Assembler::emit_deopt_handler() {
 495   // generate code for exception handler
 496   address handler_base = __ start_a_stub(deopt_handler_size());
 497   if (handler_base == nullptr) {
 498     // not enough space left for the handler
 499     bailout("deopt handler overflow");
 500     return -1;
 501   }
 502 
 503   int offset = code_offset();
 504   InternalAddress here(__ pc());
 505 
 506   __ pushptr(here.addr(), rscratch1);
 507   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 508   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 509   __ end_a_stub();
 510 
 511   return offset;
 512 }
 513 
 514 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 515   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 516   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 517     assert(result->fpu() == 0, "result must already be on TOS");
 518   }































 519 
 520   // Pop the stack before the safepoint code
 521   __ remove_frame(initial_frame_size_in_bytes());
 522 
 523   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 524     __ reserved_stack_check();
 525   }
 526 
 527   // Note: we do not need to round double result; float result has the right precision
 528   // the poll sets the condition code, but no data registers
 529 
 530 #ifdef _LP64
 531   const Register thread = r15_thread;
 532 #else
 533   const Register thread = rbx;
 534   __ get_thread(thread);
 535 #endif
 536   code_stub->set_safepoint_offset(__ offset());
 537   __ relocate(relocInfo::poll_return_type);
 538   __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
 539   __ ret(0);
 540 }
 541 
 542 




 543 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 544   guarantee(info != nullptr, "Shouldn't be null");
 545   int offset = __ offset();
 546 #ifdef _LP64
 547   const Register poll_addr = rscratch1;
 548   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 549 #else
 550   assert(tmp->is_cpu_register(), "needed");
 551   const Register poll_addr = tmp->as_register();
 552   __ get_thread(poll_addr);
 553   __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
 554 #endif
 555   add_debug_info_for_branch(info);
 556   __ relocate(relocInfo::poll_type);
 557   address pre_pc = __ pc();
 558   __ testl(rax, Address(poll_addr, 0));
 559   address post_pc = __ pc();
 560   guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
 561   return offset;
 562 }

1597     add_debug_info_for_null_check_here(op->stub()->info());
1598     __ cmpb(Address(op->klass()->as_register(),
1599                     InstanceKlass::init_state_offset()),
1600                     InstanceKlass::fully_initialized);
1601     __ jcc(Assembler::notEqual, *op->stub()->entry());
1602   }
1603   __ allocate_object(op->obj()->as_register(),
1604                      op->tmp1()->as_register(),
1605                      op->tmp2()->as_register(),
1606                      op->header_size(),
1607                      op->object_size(),
1608                      op->klass()->as_register(),
1609                      *op->stub()->entry());
1610   __ bind(*op->stub()->continuation());
1611 }
1612 
1613 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1614   Register len =  op->len()->as_register();
1615   LP64_ONLY( __ movslq(len, len); )
1616 
1617   if (UseSlowPath ||
1618       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1619       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1620     __ jmp(*op->stub()->entry());
1621   } else {
1622     Register tmp1 = op->tmp1()->as_register();
1623     Register tmp2 = op->tmp2()->as_register();
1624     Register tmp3 = op->tmp3()->as_register();
1625     if (len == tmp1) {
1626       tmp1 = tmp3;
1627     } else if (len == tmp2) {
1628       tmp2 = tmp3;
1629     } else if (len == tmp3) {
1630       // everything is ok
1631     } else {
1632       __ mov(tmp3, len);
1633     }
1634     __ allocate_array(op->obj()->as_register(),
1635                       len,
1636                       tmp1,
1637                       tmp2,

1695     assert(data != nullptr,                "need data for type check");
1696     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1697   }
1698   Label* success_target = success;
1699   Label* failure_target = failure;
1700 
1701   if (obj == k_RInfo) {
1702     k_RInfo = dst;
1703   } else if (obj == klass_RInfo) {
1704     klass_RInfo = dst;
1705   }
1706   if (k->is_loaded() && !UseCompressedClassPointers) {
1707     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1708   } else {
1709     Rtmp1 = op->tmp3()->as_register();
1710     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1711   }
1712 
1713   assert_different_registers(obj, k_RInfo, klass_RInfo);
1714 
1715   __ testptr(obj, obj);
1716   if (op->should_profile()) {
1717     Label not_null;
1718     Register mdo  = klass_RInfo;
1719     __ mov_metadata(mdo, md->constant_encoding());
1720     __ jccb(Assembler::notEqual, not_null);
1721     // Object is null; update MDO and exit
1722     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1723     int header_bits = BitData::null_seen_byte_constant();
1724     __ orb(data_addr, header_bits);
1725     __ jmp(*obj_is_null);
1726     __ bind(not_null);
1727 
1728     Label update_done;
1729     Register recv = k_RInfo;
1730     __ load_klass(recv, obj, tmp_load_klass);
1731     type_profile_helper(mdo, md, data, recv, &update_done);
1732 
1733     Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1734     __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1735 
1736     __ bind(update_done);
1737   } else {
1738     __ jcc(Assembler::equal, *obj_is_null);


1739   }
1740 
1741   if (!k->is_loaded()) {
1742     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1743   } else {
1744 #ifdef _LP64
1745     __ mov_metadata(k_RInfo, k->constant_encoding());
1746 #endif // _LP64
1747   }
1748   __ verify_oop(obj);
1749 
1750   if (op->fast_check()) {
1751     // get object class
1752     // not a safepoint as obj null check happens earlier
1753 #ifdef _LP64
1754     if (UseCompressedClassPointers) {
1755       __ load_klass(Rtmp1, obj, tmp_load_klass);
1756       __ cmpptr(k_RInfo, Rtmp1);
1757     } else {
1758       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));

1910         __ mov(dst, obj);
1911       }
1912     } else
1913       if (code == lir_instanceof) {
1914         Register obj = op->object()->as_register();
1915         Register dst = op->result_opr()->as_register();
1916         Label success, failure, done;
1917         emit_typecheck_helper(op, &success, &failure, &failure);
1918         __ bind(failure);
1919         __ xorptr(dst, dst);
1920         __ jmpb(done);
1921         __ bind(success);
1922         __ movptr(dst, 1);
1923         __ bind(done);
1924       } else {
1925         ShouldNotReachHere();
1926       }
1927 
1928 }
1929 




























































































































1930 
1931 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1932   if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) {
1933     assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1934     assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1935     assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1936     assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1937     Register addr = op->addr()->as_register();
1938     __ lock();
1939     NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1940 
1941   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1942     NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1943     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1944     Register newval = op->new_value()->as_register();
1945     Register cmpval = op->cmp_value()->as_register();
1946     assert(cmpval == rax, "wrong register");
1947     assert(newval != noreg, "new val must be register");
1948     assert(cmpval != newval, "cmp and new values must be in different registers");
1949     assert(cmpval != addr, "cmp and addr must be in different registers");

1970       __ cmpxchgl(newval, Address(addr, 0));
1971     }
1972 #ifdef _LP64
1973   } else if (op->code() == lir_cas_long) {
1974     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1975     Register newval = op->new_value()->as_register_lo();
1976     Register cmpval = op->cmp_value()->as_register_lo();
1977     assert(cmpval == rax, "wrong register");
1978     assert(newval != noreg, "new val must be register");
1979     assert(cmpval != newval, "cmp and new values must be in different registers");
1980     assert(cmpval != addr, "cmp and addr must be in different registers");
1981     assert(newval != addr, "new value and addr must be in different registers");
1982     __ lock();
1983     __ cmpxchgq(newval, Address(addr, 0));
1984 #endif // _LP64
1985   } else {
1986     Unimplemented();
1987   }
1988 }
1989 















1990 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1991                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1992   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1993 
1994   Assembler::Condition acond, ncond;
1995   switch (condition) {
1996     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1997     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1998     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1999     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
2000     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
2001     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
2002     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
2003     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
2004     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
2005                                 ShouldNotReachHere();
2006   }
2007 
2008   if (opr1->is_cpu_register()) {
2009     reg2reg(opr1, result);

2847   int offset = __ offset();
2848   switch (code) {
2849   case lir_static_call:
2850   case lir_optvirtual_call:
2851   case lir_dynamic_call:
2852     offset += NativeCall::displacement_offset;
2853     break;
2854   case lir_icvirtual_call:
2855     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size;
2856     break;
2857   default: ShouldNotReachHere();
2858   }
2859   __ align(BytesPerWord, offset);
2860 }
2861 
2862 
2863 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2864   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2865          "must be aligned");
2866   __ call(AddressLiteral(op->addr(), rtype));
2867   add_call_info(code_offset(), op->info());
2868   __ post_call_nop();
2869 }
2870 
2871 
2872 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2873   __ ic_call(op->addr());
2874   add_call_info(code_offset(), op->info());
2875   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2876          "must be aligned");
2877   __ post_call_nop();
2878 }
2879 
2880 
2881 void LIR_Assembler::emit_static_call_stub() {
2882   address call_pc = __ pc();
2883   address stub = __ start_a_stub(call_stub_size());
2884   if (stub == nullptr) {
2885     bailout("static call stub overflow");
2886     return;
2887   }
2888 
2889   int start = __ offset();
2890 
2891   // make sure that the displacement word of the call ends up word aligned
2892   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset);
2893   __ relocate(static_stub_Relocation::spec(call_pc));
2894   __ mov_metadata(rbx, (Metadata*)nullptr);

3035   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3036 }
3037 
3038 
3039 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
3040   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3041   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3042   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3043   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
3044 }
3045 
3046 
3047 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
3048   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3049   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3050   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3051   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
3052 }
3053 
3054 


























3055 // This code replaces a call to arraycopy; no exception may
3056 // be thrown in this code, they must be thrown in the System.arraycopy
3057 // activation frame; we could save some checks if this would not be the case
3058 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3059   ciArrayKlass* default_type = op->expected_type();
3060   Register src = op->src()->as_register();
3061   Register dst = op->dst()->as_register();
3062   Register src_pos = op->src_pos()->as_register();
3063   Register dst_pos = op->dst_pos()->as_register();
3064   Register length  = op->length()->as_register();
3065   Register tmp = op->tmp()->as_register();
3066   Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3067 
3068   CodeStub* stub = op->stub();
3069   int flags = op->flags();
3070   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
3071   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
3072 






3073   // if we don't know anything, just go through the generic arraycopy
3074   if (default_type == nullptr) {
3075     // save outgoing arguments on stack in case call to System.arraycopy is needed
3076     // HACK ALERT. This code used to push the parameters in a hardwired fashion
3077     // for interpreter calling conventions. Now we have to do it in new style conventions.
3078     // For the moment until C1 gets the new register allocator I just force all the
3079     // args to the right place (except the register args) and then on the back side
3080     // reload the register args properly if we go slow path. Yuck
3081 
3082     // These are proper for the calling convention
3083     store_parameter(length, 2);
3084     store_parameter(dst_pos, 1);
3085     store_parameter(dst, 0);
3086 
3087     // these are just temporary placements until we need to reload
3088     store_parameter(src_pos, 3);
3089     store_parameter(src, 4);
3090     NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3091 
3092     address copyfunc_addr = StubRoutines::generic_arraycopy();

3146     __ mov(tmp, rax);
3147     __ xorl(tmp, -1);
3148 
3149     // Reload values from the stack so they are where the stub
3150     // expects them.
3151     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3152     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3153     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3154     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3155     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3156 
3157     __ subl(length, tmp);
3158     __ addl(src_pos, tmp);
3159     __ addl(dst_pos, tmp);
3160     __ jmp(*stub->entry());
3161 
3162     __ bind(*stub->continuation());
3163     return;
3164   }
3165 








3166   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3167 
3168   int elem_size = type2aelembytes(basic_type);
3169   Address::ScaleFactor scale;
3170 
3171   switch (elem_size) {
3172     case 1 :
3173       scale = Address::times_1;
3174       break;
3175     case 2 :
3176       scale = Address::times_2;
3177       break;
3178     case 4 :
3179       scale = Address::times_4;
3180       break;
3181     case 8 :
3182       scale = Address::times_8;
3183       break;
3184     default:
3185       scale = Address::no_scale;

3747         __ jccb(Assembler::zero, next);
3748 #endif
3749         // first time here. Set profile type.
3750         __ movptr(mdo_addr, tmp);
3751       } else {
3752         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3753                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3754 
3755         __ movptr(tmp, mdo_addr);
3756         __ testptr(tmp, TypeEntries::type_unknown);
3757         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3758 
3759         __ orptr(mdo_addr, TypeEntries::type_unknown);
3760       }
3761     }
3762 
3763     __ bind(next);
3764   }
3765 }
3766 




















3767 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3768   Unimplemented();
3769 }
3770 
3771 
3772 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3773   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3774 }
3775 
3776 
3777 void LIR_Assembler::align_backward_branch_target() {
3778   __ align(BytesPerWord);
3779 }
3780 
3781 
3782 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3783   if (left->is_single_cpu()) {
3784     __ negl(left->as_register());
3785     move_regs(left->as_register(), dest->as_register());
3786 

4010 }
4011 
4012 void LIR_Assembler::membar_storeload() {
4013   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
4014 }
4015 
4016 void LIR_Assembler::on_spin_wait() {
4017   __ pause ();
4018 }
4019 
4020 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
4021   assert(result_reg->is_register(), "check");
4022 #ifdef _LP64
4023   // __ get_thread(result_reg->as_register_lo());
4024   __ mov(result_reg->as_register(), r15_thread);
4025 #else
4026   __ get_thread(result_reg->as_register());
4027 #endif // _LP64
4028 }
4029 



4030 
4031 void LIR_Assembler::peephole(LIR_List*) {
4032   // do nothing for now
4033 }
4034 
4035 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
4036   assert(data == dest, "xchg/xadd uses only 2 operands");
4037 
4038   if (data->type() == T_INT) {
4039     if (code == lir_xadd) {
4040       __ lock();
4041       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
4042     } else {
4043       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
4044     }
4045   } else if (data->is_oop()) {
4046     assert (code == lir_xchg, "xadd for oops");
4047     Register obj = data->as_register();
4048 #ifdef _LP64
4049     if (UseCompressedOops) {

  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "c1/c1_CodeStubs.hpp"
  29 #include "c1/c1_Compilation.hpp"
  30 #include "c1/c1_LIRAssembler.hpp"
  31 #include "c1/c1_MacroAssembler.hpp"
  32 #include "c1/c1_Runtime1.hpp"
  33 #include "c1/c1_ValueStack.hpp"
  34 #include "ci/ciArrayKlass.hpp"
  35 #include "ci/ciInlineKlass.hpp"
  36 #include "ci/ciInstance.hpp"
  37 #include "compiler/oopMap.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "gc/shared/gc_globals.hpp"
  40 #include "nativeInst_x86.hpp"
  41 #include "oops/oop.inline.hpp"
  42 #include "oops/objArrayKlass.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/safepointMechanism.hpp"
  45 #include "runtime/sharedRuntime.hpp"
  46 #include "runtime/stubRoutines.hpp"
  47 #include "utilities/powerOfTwo.hpp"
  48 #include "vmreg_x86.inline.hpp"
  49 
  50 
  51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  52 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  53 // fast versions of NegF/NegD and AbsF/AbsD.
  54 
  55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  57   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  58   // of 128-bits operands for SSE instructions.
  59   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  60   // Store the value to a 128-bits operand.
  61   operand[0] = lo;

 464     __ bind(*stub->continuation());
 465   }
 466 
 467   if (compilation()->env()->dtrace_method_probes()) {
 468 #ifdef _LP64
 469     __ mov(rdi, r15_thread);
 470     __ mov_metadata(rsi, method()->constant_encoding());
 471 #else
 472     __ get_thread(rax);
 473     __ movptr(Address(rsp, 0), rax);
 474     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
 475 #endif
 476     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 477   }
 478 
 479   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 480     __ mov(rax, rbx);  // Restore the exception
 481   }
 482 
 483   // remove the activation and dispatch to the unwind handler
 484   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 485   __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 486 
 487   // Emit the slow path assembly
 488   if (stub != nullptr) {
 489     stub->emit_code(this);
 490   }
 491 
 492   return offset;
 493 }
 494 
 495 
 496 int LIR_Assembler::emit_deopt_handler() {
 497   // generate code for exception handler
 498   address handler_base = __ start_a_stub(deopt_handler_size());
 499   if (handler_base == nullptr) {
 500     // not enough space left for the handler
 501     bailout("deopt handler overflow");
 502     return -1;
 503   }
 504 
 505   int offset = code_offset();
 506   InternalAddress here(__ pc());
 507 
 508   __ pushptr(here.addr(), rscratch1);
 509   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 510   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 511   __ end_a_stub();
 512 
 513   return offset;
 514 }
 515 
 516 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 517   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 518   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 519     assert(result->fpu() == 0, "result must already be on TOS");
 520   }
 521   if (InlineTypeReturnedAsFields) {
 522   #ifndef _LP64
 523      Unimplemented();
 524   #endif
 525     // Check if we are returning an non-null inline type and load its fields into registers
 526     ciType* return_type = compilation()->method()->return_type();
 527     if (return_type->is_inlinetype()) {
 528       ciInlineKlass* vk = return_type->as_inline_klass();
 529       if (vk->can_be_returned_as_fields()) {
 530         address unpack_handler = vk->unpack_handler();
 531         assert(unpack_handler != nullptr, "must be");
 532         __ call(RuntimeAddress(unpack_handler));
 533       }
 534     } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
 535       Label skip;
 536       __ test_oop_is_not_inline_type(rax, rscratch1, skip);
 537 
 538       // Load fields from a buffered value with an inline class specific handler
 539       __ load_klass(rdi, rax, rscratch1);
 540       __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 541       __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
 542       // Unpack handler can be null if inline type is not scalarizable in returns
 543       __ testptr(rdi, rdi);
 544       __ jcc(Assembler::zero, skip);
 545       __ call(rdi);
 546 
 547       __ bind(skip);
 548     }
 549     // At this point, rax points to the value object (for interpreter or C1 caller).
 550     // The fields of the object are copied into registers (for C2 caller).
 551   }
 552 
 553   // Pop the stack before the safepoint code
 554   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 555 
 556   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 557     __ reserved_stack_check();
 558   }
 559 
 560   // Note: we do not need to round double result; float result has the right precision
 561   // the poll sets the condition code, but no data registers
 562 
 563 #ifdef _LP64
 564   const Register thread = r15_thread;
 565 #else
 566   const Register thread = rbx;
 567   __ get_thread(thread);
 568 #endif
 569   code_stub->set_safepoint_offset(__ offset());
 570   __ relocate(relocInfo::poll_return_type);
 571   __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
 572   __ ret(0);
 573 }
 574 
 575 
 576 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
 577   return (__ store_inline_type_fields_to_buf(vk, false));
 578 }
 579 
 580 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 581   guarantee(info != nullptr, "Shouldn't be null");
 582   int offset = __ offset();
 583 #ifdef _LP64
 584   const Register poll_addr = rscratch1;
 585   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 586 #else
 587   assert(tmp->is_cpu_register(), "needed");
 588   const Register poll_addr = tmp->as_register();
 589   __ get_thread(poll_addr);
 590   __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
 591 #endif
 592   add_debug_info_for_branch(info);
 593   __ relocate(relocInfo::poll_type);
 594   address pre_pc = __ pc();
 595   __ testl(rax, Address(poll_addr, 0));
 596   address post_pc = __ pc();
 597   guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
 598   return offset;
 599 }

1634     add_debug_info_for_null_check_here(op->stub()->info());
1635     __ cmpb(Address(op->klass()->as_register(),
1636                     InstanceKlass::init_state_offset()),
1637                     InstanceKlass::fully_initialized);
1638     __ jcc(Assembler::notEqual, *op->stub()->entry());
1639   }
1640   __ allocate_object(op->obj()->as_register(),
1641                      op->tmp1()->as_register(),
1642                      op->tmp2()->as_register(),
1643                      op->header_size(),
1644                      op->object_size(),
1645                      op->klass()->as_register(),
1646                      *op->stub()->entry());
1647   __ bind(*op->stub()->continuation());
1648 }
1649 
1650 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1651   Register len =  op->len()->as_register();
1652   LP64_ONLY( __ movslq(len, len); )
1653 
1654   if (UseSlowPath || op->is_null_free() ||
1655       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1656       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1657     __ jmp(*op->stub()->entry());
1658   } else {
1659     Register tmp1 = op->tmp1()->as_register();
1660     Register tmp2 = op->tmp2()->as_register();
1661     Register tmp3 = op->tmp3()->as_register();
1662     if (len == tmp1) {
1663       tmp1 = tmp3;
1664     } else if (len == tmp2) {
1665       tmp2 = tmp3;
1666     } else if (len == tmp3) {
1667       // everything is ok
1668     } else {
1669       __ mov(tmp3, len);
1670     }
1671     __ allocate_array(op->obj()->as_register(),
1672                       len,
1673                       tmp1,
1674                       tmp2,

1732     assert(data != nullptr,                "need data for type check");
1733     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1734   }
1735   Label* success_target = success;
1736   Label* failure_target = failure;
1737 
1738   if (obj == k_RInfo) {
1739     k_RInfo = dst;
1740   } else if (obj == klass_RInfo) {
1741     klass_RInfo = dst;
1742   }
1743   if (k->is_loaded() && !UseCompressedClassPointers) {
1744     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1745   } else {
1746     Rtmp1 = op->tmp3()->as_register();
1747     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1748   }
1749 
1750   assert_different_registers(obj, k_RInfo, klass_RInfo);
1751 
1752   if (op->need_null_check()) {
1753     __ testptr(obj, obj);
1754     if (op->should_profile()) {
1755       Label not_null;
1756       Register mdo  = klass_RInfo;
1757       __ mov_metadata(mdo, md->constant_encoding());
1758       __ jccb(Assembler::notEqual, not_null);
1759       // Object is null; update MDO and exit
1760       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1761       int header_bits = BitData::null_seen_byte_constant();
1762       __ orb(data_addr, header_bits);
1763       __ jmp(*obj_is_null);
1764       __ bind(not_null);
1765 
1766       Label update_done;
1767       Register recv = k_RInfo;
1768       __ load_klass(recv, obj, tmp_load_klass);
1769       type_profile_helper(mdo, md, data, recv, &update_done);
1770 
1771       Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1772       __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1773 
1774       __ bind(update_done);
1775     } else {
1776       __ jcc(Assembler::equal, *obj_is_null);
1777     }
1778   }
1779 
1780   if (!k->is_loaded()) {
1781     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1782   } else {
1783 #ifdef _LP64
1784     __ mov_metadata(k_RInfo, k->constant_encoding());
1785 #endif // _LP64
1786   }
1787   __ verify_oop(obj);
1788 
1789   if (op->fast_check()) {
1790     // get object class
1791     // not a safepoint as obj null check happens earlier
1792 #ifdef _LP64
1793     if (UseCompressedClassPointers) {
1794       __ load_klass(Rtmp1, obj, tmp_load_klass);
1795       __ cmpptr(k_RInfo, Rtmp1);
1796     } else {
1797       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));

1949         __ mov(dst, obj);
1950       }
1951     } else
1952       if (code == lir_instanceof) {
1953         Register obj = op->object()->as_register();
1954         Register dst = op->result_opr()->as_register();
1955         Label success, failure, done;
1956         emit_typecheck_helper(op, &success, &failure, &failure);
1957         __ bind(failure);
1958         __ xorptr(dst, dst);
1959         __ jmpb(done);
1960         __ bind(success);
1961         __ movptr(dst, 1);
1962         __ bind(done);
1963       } else {
1964         ShouldNotReachHere();
1965       }
1966 
1967 }
1968 
1969 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1970   // We are loading/storing from/to an array that *may* be a flat array (the
1971   // declared type is Object[], abstract[], interface[] or VT.ref[]).
1972   // If this array is a flat array, take the slow path.
1973   Register klass = op->tmp()->as_register();
1974   if (UseArrayMarkWordCheck) {
1975     __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1976   } else {
1977     Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1978     __ load_klass(klass, op->array()->as_register(), tmp_load_klass);
1979     __ movl(klass, Address(klass, Klass::layout_helper_offset()));
1980     __ testl(klass, Klass::_lh_array_tag_flat_value_bit_inplace);
1981     __ jcc(Assembler::notZero, *op->stub()->entry());
1982   }
1983   if (!op->value()->is_illegal()) {
1984     // The array is not a flat array, but it might be null-free. If we are storing
1985     // a null into a null-free array, take the slow path (which will throw NPE).
1986     Label skip;
1987     __ cmpptr(op->value()->as_register(), NULL_WORD);
1988     __ jcc(Assembler::notEqual, skip);
1989     if (UseArrayMarkWordCheck) {
1990       __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1991     } else {
1992       __ testl(klass, Klass::_lh_null_free_array_bit_inplace);
1993       __ jcc(Assembler::notZero, *op->stub()->entry());
1994     }
1995     __ bind(skip);
1996   }
1997 }
1998 
1999 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
2000   // We are storing into an array that *may* be null-free (the declared type is
2001   // Object[], abstract[], interface[] or VT.ref[]).
2002   if (UseArrayMarkWordCheck) {
2003     Label test_mark_word;
2004     Register tmp = op->tmp()->as_register();
2005     __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
2006     __ testl(tmp, markWord::unlocked_value);
2007     __ jccb(Assembler::notZero, test_mark_word);
2008     __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
2009     __ bind(test_mark_word);
2010     __ testl(tmp, markWord::null_free_array_bit_in_place);
2011   } else {
2012     Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
2013     Register klass = op->tmp()->as_register();
2014     __ load_klass(klass, op->array()->as_register(), tmp_load_klass);
2015     __ movl(klass, Address(klass, Klass::layout_helper_offset()));
2016     __ testl(klass, Klass::_lh_null_free_array_bit_inplace);
2017   }
2018 }
2019 
2020 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
2021   Label L_oops_equal;
2022   Label L_oops_not_equal;
2023   Label L_end;
2024 
2025   Register left  = op->left()->as_register();
2026   Register right = op->right()->as_register();
2027 
2028   __ cmpptr(left, right);
2029   __ jcc(Assembler::equal, L_oops_equal);
2030 
2031   // (1) Null check -- if one of the operands is null, the other must not be null (because
2032   //     the two references are not equal), so they are not substitutable,
2033   //     FIXME: do null check only if the operand is nullable
2034   __ testptr(left, right);
2035   __ jcc(Assembler::zero, L_oops_not_equal);
2036 
2037   ciKlass* left_klass = op->left_klass();
2038   ciKlass* right_klass = op->right_klass();
2039 
2040   // (2) Inline type check -- if either of the operands is not a inline type,
2041   //     they are not substitutable. We do this only if we are not sure that the
2042   //     operands are inline type
2043   if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
2044       !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
2045     Register tmp1  = op->tmp1()->as_register();
2046     __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
2047     __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
2048     __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
2049     __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
2050     __ jcc(Assembler::notEqual, L_oops_not_equal);
2051   }
2052 
2053   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
2054   if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
2055     // No need to load klass -- the operands are statically known to be the same inline klass.
2056     __ jmp(*op->stub()->entry());
2057   } else {
2058     Register left_klass_op = op->left_klass_op()->as_register();
2059     Register right_klass_op = op->right_klass_op()->as_register();
2060 
2061     if (UseCompressedClassPointers) {
2062       __ movl(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
2063       __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2064       __ cmpl(left_klass_op, right_klass_op);
2065     } else {
2066       __ movptr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
2067       __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2068       __ cmpptr(left_klass_op, right_klass_op);
2069     }
2070 
2071     __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
2072     // fall through to L_oops_not_equal
2073   }
2074 
2075   __ bind(L_oops_not_equal);
2076   move(op->not_equal_result(), op->result_opr());
2077   __ jmp(L_end);
2078 
2079   __ bind(L_oops_equal);
2080   move(op->equal_result(), op->result_opr());
2081   __ jmp(L_end);
2082 
2083   // We've returned from the stub. RAX contains 0x0 IFF the two
2084   // operands are not substitutable. (Don't compare against 0x1 in case the
2085   // C compiler is naughty)
2086   __ bind(*op->stub()->continuation());
2087   __ cmpl(rax, 0);
2088   __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
2089   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
2090   // fall-through
2091   __ bind(L_end);
2092 }
2093 
2094 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2095   if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) {
2096     assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
2097     assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
2098     assert(op->new_value()->as_register_lo() == rbx, "wrong register");
2099     assert(op->new_value()->as_register_hi() == rcx, "wrong register");
2100     Register addr = op->addr()->as_register();
2101     __ lock();
2102     NOT_LP64(__ cmpxchg8(Address(addr, 0)));
2103 
2104   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
2105     NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
2106     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2107     Register newval = op->new_value()->as_register();
2108     Register cmpval = op->cmp_value()->as_register();
2109     assert(cmpval == rax, "wrong register");
2110     assert(newval != noreg, "new val must be register");
2111     assert(cmpval != newval, "cmp and new values must be in different registers");
2112     assert(cmpval != addr, "cmp and addr must be in different registers");

2133       __ cmpxchgl(newval, Address(addr, 0));
2134     }
2135 #ifdef _LP64
2136   } else if (op->code() == lir_cas_long) {
2137     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2138     Register newval = op->new_value()->as_register_lo();
2139     Register cmpval = op->cmp_value()->as_register_lo();
2140     assert(cmpval == rax, "wrong register");
2141     assert(newval != noreg, "new val must be register");
2142     assert(cmpval != newval, "cmp and new values must be in different registers");
2143     assert(cmpval != addr, "cmp and addr must be in different registers");
2144     assert(newval != addr, "new value and addr must be in different registers");
2145     __ lock();
2146     __ cmpxchgq(newval, Address(addr, 0));
2147 #endif // _LP64
2148   } else {
2149     Unimplemented();
2150   }
2151 }
2152 
2153 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
2154   assert(dst->is_cpu_register(), "must be");
2155   assert(dst->type() == src->type(), "must be");
2156 
2157   if (src->is_cpu_register()) {
2158     reg2reg(src, dst);
2159   } else if (src->is_stack()) {
2160     stack2reg(src, dst, dst->type());
2161   } else if (src->is_constant()) {
2162     const2reg(src, dst, lir_patch_none, nullptr);
2163   } else {
2164     ShouldNotReachHere();
2165   }
2166 }
2167 
2168 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
2169                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
2170   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
2171 
2172   Assembler::Condition acond, ncond;
2173   switch (condition) {
2174     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
2175     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
2176     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
2177     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
2178     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
2179     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
2180     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
2181     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
2182     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
2183                                 ShouldNotReachHere();
2184   }
2185 
2186   if (opr1->is_cpu_register()) {
2187     reg2reg(opr1, result);

3025   int offset = __ offset();
3026   switch (code) {
3027   case lir_static_call:
3028   case lir_optvirtual_call:
3029   case lir_dynamic_call:
3030     offset += NativeCall::displacement_offset;
3031     break;
3032   case lir_icvirtual_call:
3033     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size;
3034     break;
3035   default: ShouldNotReachHere();
3036   }
3037   __ align(BytesPerWord, offset);
3038 }
3039 
3040 
3041 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
3042   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
3043          "must be aligned");
3044   __ call(AddressLiteral(op->addr(), rtype));
3045   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
3046   __ post_call_nop();
3047 }
3048 
3049 
3050 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
3051   __ ic_call(op->addr());
3052   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
3053   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
3054          "must be aligned");
3055   __ post_call_nop();
3056 }
3057 
3058 
3059 void LIR_Assembler::emit_static_call_stub() {
3060   address call_pc = __ pc();
3061   address stub = __ start_a_stub(call_stub_size());
3062   if (stub == nullptr) {
3063     bailout("static call stub overflow");
3064     return;
3065   }
3066 
3067   int start = __ offset();
3068 
3069   // make sure that the displacement word of the call ends up word aligned
3070   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset);
3071   __ relocate(static_stub_Relocation::spec(call_pc));
3072   __ mov_metadata(rbx, (Metadata*)nullptr);

3213   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3214 }
3215 
3216 
3217 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
3218   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3219   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3220   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3221   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
3222 }
3223 
3224 
3225 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
3226   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3227   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3228   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3229   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
3230 }
3231 
3232 
3233 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
3234   if (null_check) {
3235     __ testptr(obj, obj);
3236     __ jcc(Assembler::zero, *slow_path->entry());
3237   }
3238   if (UseArrayMarkWordCheck) {
3239     if (is_dest) {
3240       __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
3241     } else {
3242       __ test_flat_array_oop(obj, tmp, *slow_path->entry());
3243     }
3244   } else {
3245     Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3246     __ load_klass(tmp, obj, tmp_load_klass);
3247     __ movl(tmp, Address(tmp, Klass::layout_helper_offset()));
3248     if (is_dest) {
3249       // Take the slow path if it's a null_free destination array, in case the source array contains nullptrs.
3250       __ testl(tmp, Klass::_lh_null_free_array_bit_inplace);
3251     } else {
3252       __ testl(tmp, Klass::_lh_array_tag_flat_value_bit_inplace);
3253     }
3254     __ jcc(Assembler::notZero, *slow_path->entry());
3255   }
3256 }
3257 
3258 
3259 // This code replaces a call to arraycopy; no exception may
3260 // be thrown in this code, they must be thrown in the System.arraycopy
3261 // activation frame; we could save some checks if this would not be the case
3262 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3263   ciArrayKlass* default_type = op->expected_type();
3264   Register src = op->src()->as_register();
3265   Register dst = op->dst()->as_register();
3266   Register src_pos = op->src_pos()->as_register();
3267   Register dst_pos = op->dst_pos()->as_register();
3268   Register length  = op->length()->as_register();
3269   Register tmp = op->tmp()->as_register();
3270   Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3271 
3272   CodeStub* stub = op->stub();
3273   int flags = op->flags();
3274   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
3275   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
3276 
3277   if (flags & LIR_OpArrayCopy::always_slow_path) {
3278     __ jmp(*stub->entry());
3279     __ bind(*stub->continuation());
3280     return;
3281   }
3282 
3283   // if we don't know anything, just go through the generic arraycopy
3284   if (default_type == nullptr) {
3285     // save outgoing arguments on stack in case call to System.arraycopy is needed
3286     // HACK ALERT. This code used to push the parameters in a hardwired fashion
3287     // for interpreter calling conventions. Now we have to do it in new style conventions.
3288     // For the moment until C1 gets the new register allocator I just force all the
3289     // args to the right place (except the register args) and then on the back side
3290     // reload the register args properly if we go slow path. Yuck
3291 
3292     // These are proper for the calling convention
3293     store_parameter(length, 2);
3294     store_parameter(dst_pos, 1);
3295     store_parameter(dst, 0);
3296 
3297     // these are just temporary placements until we need to reload
3298     store_parameter(src_pos, 3);
3299     store_parameter(src, 4);
3300     NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3301 
3302     address copyfunc_addr = StubRoutines::generic_arraycopy();

3356     __ mov(tmp, rax);
3357     __ xorl(tmp, -1);
3358 
3359     // Reload values from the stack so they are where the stub
3360     // expects them.
3361     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3362     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3363     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3364     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3365     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3366 
3367     __ subl(length, tmp);
3368     __ addl(src_pos, tmp);
3369     __ addl(dst_pos, tmp);
3370     __ jmp(*stub->entry());
3371 
3372     __ bind(*stub->continuation());
3373     return;
3374   }
3375 
3376   // Handle inline type arrays
3377   if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
3378     arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
3379   }
3380   if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
3381     arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
3382   }
3383 
3384   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3385 
3386   int elem_size = type2aelembytes(basic_type);
3387   Address::ScaleFactor scale;
3388 
3389   switch (elem_size) {
3390     case 1 :
3391       scale = Address::times_1;
3392       break;
3393     case 2 :
3394       scale = Address::times_2;
3395       break;
3396     case 4 :
3397       scale = Address::times_4;
3398       break;
3399     case 8 :
3400       scale = Address::times_8;
3401       break;
3402     default:
3403       scale = Address::no_scale;

3965         __ jccb(Assembler::zero, next);
3966 #endif
3967         // first time here. Set profile type.
3968         __ movptr(mdo_addr, tmp);
3969       } else {
3970         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3971                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3972 
3973         __ movptr(tmp, mdo_addr);
3974         __ testptr(tmp, TypeEntries::type_unknown);
3975         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3976 
3977         __ orptr(mdo_addr, TypeEntries::type_unknown);
3978       }
3979     }
3980 
3981     __ bind(next);
3982   }
3983 }
3984 
3985 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3986   Register obj = op->obj()->as_register();
3987   Register tmp = op->tmp()->as_pointer_register();
3988   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3989   bool not_null = op->not_null();
3990   int flag = op->flag();
3991 
3992   Label not_inline_type;
3993   if (!not_null) {
3994     __ testptr(obj, obj);
3995     __ jccb(Assembler::zero, not_inline_type);
3996   }
3997 
3998   __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3999 
4000   __ orb(mdo_addr, flag);
4001 
4002   __ bind(not_inline_type);
4003 }
4004 
4005 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
4006   Unimplemented();
4007 }
4008 
4009 
4010 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
4011   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
4012 }
4013 
4014 
4015 void LIR_Assembler::align_backward_branch_target() {
4016   __ align(BytesPerWord);
4017 }
4018 
4019 
4020 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
4021   if (left->is_single_cpu()) {
4022     __ negl(left->as_register());
4023     move_regs(left->as_register(), dest->as_register());
4024 

4248 }
4249 
4250 void LIR_Assembler::membar_storeload() {
4251   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
4252 }
4253 
4254 void LIR_Assembler::on_spin_wait() {
4255   __ pause ();
4256 }
4257 
4258 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
4259   assert(result_reg->is_register(), "check");
4260 #ifdef _LP64
4261   // __ get_thread(result_reg->as_register_lo());
4262   __ mov(result_reg->as_register(), r15_thread);
4263 #else
4264   __ get_thread(result_reg->as_register());
4265 #endif // _LP64
4266 }
4267 
4268 void LIR_Assembler::check_orig_pc() {
4269   __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
4270 }
4271 
4272 void LIR_Assembler::peephole(LIR_List*) {
4273   // do nothing for now
4274 }
4275 
4276 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
4277   assert(data == dest, "xchg/xadd uses only 2 operands");
4278 
4279   if (data->type() == T_INT) {
4280     if (code == lir_xadd) {
4281       __ lock();
4282       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
4283     } else {
4284       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
4285     }
4286   } else if (data->is_oop()) {
4287     assert (code == lir_xchg, "xadd for oops");
4288     Register obj = data->as_register();
4289 #ifdef _LP64
4290     if (UseCompressedOops) {
< prev index next >