1 /*
   2  * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInlineKlass.hpp"
  35 #include "ci/ciInstance.hpp"
  36 #include "ci/ciObjArrayKlass.hpp"
  37 #include "code/aotCodeCache.hpp"
  38 #include "compiler/oopMap.hpp"
  39 #include "gc/shared/collectedHeap.hpp"
  40 #include "gc/shared/gc_globals.hpp"
  41 #include "nativeInst_x86.hpp"
  42 #include "oops/oop.inline.hpp"
  43 #include "oops/objArrayKlass.hpp"
  44 #include "runtime/frame.inline.hpp"
  45 #include "runtime/safepointMechanism.hpp"
  46 #include "runtime/sharedRuntime.hpp"
  47 #include "runtime/stubRoutines.hpp"
  48 #include "runtime/threadIdentifier.hpp"
  49 #include "utilities/powerOfTwo.hpp"
  50 #include "vmreg_x86.inline.hpp"
  51 
  52 
  53 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  54 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  55 // fast versions of NegF/NegD and AbsF/AbsD.
  56 
  57 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  58 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  59   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  60   // of 128-bits operands for SSE instructions.
  61   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  62   // Store the value to a 128-bits operand.
  63   operand[0] = lo;
  64   operand[1] = hi;
  65   return operand;
  66 }
  67 
  68 // Buffer for 128-bits masks used by SSE instructions.
  69 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
  70 
  71 // Static initialization during VM startup.
  72 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2],         CONST64(0x7FFFFFFF7FFFFFFF),         CONST64(0x7FFFFFFF7FFFFFFF));
  73 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2],         CONST64(0x7FFFFFFFFFFFFFFF),         CONST64(0x7FFFFFFFFFFFFFFF));
  74 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
  75 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
  76 
  77 #if INCLUDE_CDS
  78 // publish external addresses defined in this file
  79 void LIR_Assembler::init_AOTAddressTable(GrowableArray<address>& external_addresses) {
  80 #define ADD(addr) external_addresses.append((address)(addr));
  81   ADD(float_signmask_pool);
  82   ADD(double_signmask_pool);
  83   ADD(float_signflip_pool);
  84   ADD(double_signflip_pool);
  85 #undef ADD
  86 }
  87 #endif // INCLUDE_CDS
  88 
  89 NEEDS_CLEANUP // remove this definitions ?
  90 const Register SYNC_header = rax;   // synchronization header
  91 const Register SHIFT_count = rcx;   // where count for shift operations must be
  92 
  93 #define __ _masm->
  94 
  95 static void select_different_registers(Register preserve,
  96                                        Register extra,
  97                                        Register &tmp1,
  98                                        Register &tmp2,
  99                                        Register &tmp3) {
 100   if (tmp1 == preserve) {
 101     assert_different_registers(tmp1, tmp2, tmp3, extra);
 102     tmp1 = extra;
 103   } else if (tmp2 == preserve) {
 104     assert_different_registers(tmp1, tmp2, tmp3, extra);
 105     tmp2 = extra;
 106   } else if (tmp3 == preserve) {
 107     assert_different_registers(tmp1, tmp2, tmp3, extra);
 108     tmp3 = extra;
 109   }
 110   assert_different_registers(preserve, tmp1, tmp2, tmp3);
 111 }
 112 
 113 
 114 
 115 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
 116   if (opr->is_constant()) {
 117     LIR_Const* constant = opr->as_constant_ptr();
 118     switch (constant->type()) {
 119       case T_INT: {
 120         return true;
 121       }
 122 
 123       default:
 124         return false;
 125     }
 126   }
 127   return false;
 128 }
 129 
 130 
 131 LIR_Opr LIR_Assembler::receiverOpr() {
 132   return FrameMap::receiver_opr;
 133 }
 134 
 135 LIR_Opr LIR_Assembler::osrBufferPointer() {
 136   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 137 }
 138 
 139 //--------------fpu register translations-----------------------
 140 
 141 
 142 address LIR_Assembler::float_constant(float f) {
 143   address const_addr = __ float_constant(f);
 144   if (const_addr == nullptr) {
 145     bailout("const section overflow");
 146     return __ code()->consts()->start();
 147   } else {
 148     return const_addr;
 149   }
 150 }
 151 
 152 
 153 address LIR_Assembler::double_constant(double d) {
 154   address const_addr = __ double_constant(d);
 155   if (const_addr == nullptr) {
 156     bailout("const section overflow");
 157     return __ code()->consts()->start();
 158   } else {
 159     return const_addr;
 160   }
 161 }
 162 
 163 void LIR_Assembler::breakpoint() {
 164   __ int3();
 165 }
 166 
 167 void LIR_Assembler::push(LIR_Opr opr) {
 168   if (opr->is_single_cpu()) {
 169     __ push_reg(opr->as_register());
 170   } else if (opr->is_double_cpu()) {
 171     __ push_reg(opr->as_register_lo());
 172   } else if (opr->is_stack()) {
 173     __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
 174   } else if (opr->is_constant()) {
 175     LIR_Const* const_opr = opr->as_constant_ptr();
 176     if (const_opr->type() == T_OBJECT) {
 177       __ push_oop(const_opr->as_jobject(), rscratch1);
 178     } else if (const_opr->type() == T_INT) {
 179       __ push_jint(const_opr->as_jint());
 180     } else {
 181       ShouldNotReachHere();
 182     }
 183 
 184   } else {
 185     ShouldNotReachHere();
 186   }
 187 }
 188 
 189 void LIR_Assembler::pop(LIR_Opr opr) {
 190   if (opr->is_single_cpu()) {
 191     __ pop_reg(opr->as_register());
 192   } else {
 193     ShouldNotReachHere();
 194   }
 195 }
 196 
 197 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
 198   return addr->base()->is_illegal() && addr->index()->is_illegal();
 199 }
 200 
 201 //-------------------------------------------
 202 
 203 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 204   return as_Address(addr, rscratch1);
 205 }
 206 
 207 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 208   if (addr->base()->is_illegal()) {
 209     assert(addr->index()->is_illegal(), "must be illegal too");
 210     AddressLiteral laddr((address)addr->disp(), relocInfo::none);
 211     if (! __ reachable(laddr)) {
 212       __ movptr(tmp, laddr.addr());
 213       Address res(tmp, 0);
 214       return res;
 215     } else {
 216       return __ as_Address(laddr);
 217     }
 218   }
 219 
 220   Register base = addr->base()->as_pointer_register();
 221 
 222   if (addr->index()->is_illegal()) {
 223     return Address( base, addr->disp());
 224   } else if (addr->index()->is_cpu_register()) {
 225     Register index = addr->index()->as_pointer_register();
 226     return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
 227   } else if (addr->index()->is_constant()) {
 228     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 229     assert(Assembler::is_simm32(addr_offset), "must be");
 230 
 231     return Address(base, addr_offset);
 232   } else {
 233     Unimplemented();
 234     return Address();
 235   }
 236 }
 237 
 238 
 239 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 240   Address base = as_Address(addr);
 241   return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
 242 }
 243 
 244 
 245 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 246   return as_Address(addr);
 247 }
 248 
 249 
 250 void LIR_Assembler::osr_entry() {
 251   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 252   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 253   ValueStack* entry_state = osr_entry->state();
 254   int number_of_locks = entry_state->locks_size();
 255 
 256   // we jump here if osr happens with the interpreter
 257   // state set up to continue at the beginning of the
 258   // loop that triggered osr - in particular, we have
 259   // the following registers setup:
 260   //
 261   // rcx: osr buffer
 262   //
 263 
 264   // build frame
 265   ciMethod* m = compilation()->method();
 266   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 267 
 268   // OSR buffer is
 269   //
 270   // locals[nlocals-1..0]
 271   // monitors[0..number_of_locks]
 272   //
 273   // locals is a direct copy of the interpreter frame so in the osr buffer
 274   // so first slot in the local array is the last local from the interpreter
 275   // and last slot is local[0] (receiver) from the interpreter
 276   //
 277   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 278   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 279   // in the interpreter frame (the method lock if a sync method)
 280 
 281   // Initialize monitors in the compiled activation.
 282   //   rcx: pointer to osr buffer
 283   //
 284   // All other registers are dead at this point and the locals will be
 285   // copied into place by code emitted in the IR.
 286 
 287   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 288   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 289     int monitor_offset = BytesPerWord * method()->max_locals() +
 290       (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
 291     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 292     // the OSR buffer using 2 word entries: first the lock and then
 293     // the oop.
 294     for (int i = 0; i < number_of_locks; i++) {
 295       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 296 #ifdef ASSERT
 297       // verify the interpreter's monitor has a non-null object
 298       {
 299         Label L;
 300         __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
 301         __ jcc(Assembler::notZero, L);
 302         __ stop("locked object is null");
 303         __ bind(L);
 304       }
 305 #endif
 306       __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
 307       __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
 308       __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 309       __ movptr(frame_map()->address_for_monitor_object(i), rbx);
 310     }
 311   }
 312 }
 313 
 314 
 315 // inline cache check; done before the frame is built.
 316 int LIR_Assembler::check_icache() {
 317   return __ ic_check(CodeEntryAlignment);
 318 }
 319 
 320 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 321   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 322   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 323 
 324   Label L_skip_barrier;
 325   Register klass = rscratch1;
 326 
 327   __ mov_metadata(klass, method->holder()->constant_encoding());
 328   __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
 329 
 330   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 331 
 332   __ bind(L_skip_barrier);
 333 }
 334 
 335 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
 336   jobject o = nullptr;
 337   PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
 338   __ movoop(reg, o);
 339   patching_epilog(patch, lir_patch_normal, reg, info);
 340 }
 341 
 342 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 343   Metadata* o = nullptr;
 344   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 345   __ mov_metadata(reg, o);
 346   patching_epilog(patch, lir_patch_normal, reg, info);
 347 }
 348 
 349 // This specifies the rsp decrement needed to build the frame
 350 int LIR_Assembler::initial_frame_size_in_bytes() const {
 351   // if rounding, must let FrameMap know!
 352 
 353   // The frame_map records size in slots (32bit word)
 354 
 355   // subtract two words to account for return address and link
 356   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 357 }
 358 
 359 
 360 int LIR_Assembler::emit_exception_handler() {
 361   // generate code for exception handler
 362   address handler_base = __ start_a_stub(exception_handler_size());
 363   if (handler_base == nullptr) {
 364     // not enough space left for the handler
 365     bailout("exception handler overflow");
 366     return -1;
 367   }
 368 
 369   int offset = code_offset();
 370 
 371   // the exception oop and pc are in rax, and rdx
 372   // no other registers need to be preserved, so invalidate them
 373   __ invalidate_registers(false, true, true, false, true, true);
 374 
 375   // check that there is really an exception
 376   __ verify_not_null_oop(rax);
 377 
 378   // search an exception handler (rax: exception oop, rdx: throwing pc)
 379   __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
 380   __ should_not_reach_here();
 381   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 382   __ end_a_stub();
 383 
 384   return offset;
 385 }
 386 
 387 
 388 // Emit the code to remove the frame from the stack in the exception
 389 // unwind path.
 390 int LIR_Assembler::emit_unwind_handler() {
 391 #ifndef PRODUCT
 392   if (CommentedAssembly) {
 393     _masm->block_comment("Unwind handler");
 394   }
 395 #endif
 396 
 397   int offset = code_offset();
 398 
 399   // Fetch the exception from TLS and clear out exception related thread state
 400   __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
 401   __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
 402   __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
 403 
 404   __ bind(_unwind_handler_entry);
 405   __ verify_not_null_oop(rax);
 406   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 407     __ mov(rbx, rax);  // Preserve the exception (rbx is always callee-saved)
 408   }
 409 
 410   // Perform needed unlocking
 411   MonitorExitStub* stub = nullptr;
 412   if (method()->is_synchronized()) {
 413     monitor_address(0, FrameMap::rax_opr);
 414     stub = new MonitorExitStub(FrameMap::rax_opr, 0);
 415     __ unlock_object(rdi, rsi, rax, *stub->entry());
 416     __ bind(*stub->continuation());
 417   }
 418 
 419   if (compilation()->env()->dtrace_method_probes()) {
 420     __ mov(rdi, r15_thread);
 421     __ mov_metadata(rsi, method()->constant_encoding());
 422     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 423   }
 424 
 425   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 426     __ mov(rax, rbx);  // Restore the exception
 427   }
 428 
 429   // remove the activation and dispatch to the unwind handler
 430   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 431   __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
 432 
 433   // Emit the slow path assembly
 434   if (stub != nullptr) {
 435     stub->emit_code(this);
 436   }
 437 
 438   return offset;
 439 }
 440 
 441 
 442 int LIR_Assembler::emit_deopt_handler() {
 443   // generate code for exception handler
 444   address handler_base = __ start_a_stub(deopt_handler_size());
 445   if (handler_base == nullptr) {
 446     // not enough space left for the handler
 447     bailout("deopt handler overflow");
 448     return -1;
 449   }
 450 
 451   int offset = code_offset();
 452 
 453   Label start;
 454   __ bind(start);
 455 
 456   __ call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 457 
 458   int entry_offset = __ offset();
 459 
 460   __ jmp(start);
 461 
 462   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 463   assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size,
 464          "out of bounds read in post-call NOP check");
 465   __ end_a_stub();
 466 
 467   return entry_offset;
 468 }
 469 
 470 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 471   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 472   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 473     assert(result->fpu() == 0, "result must already be on TOS");
 474   }
 475   if (InlineTypeReturnedAsFields) {
 476   #ifndef _LP64
 477      Unimplemented();
 478   #endif
 479     // Check if we are returning an non-null inline type and load its fields into registers
 480     ciType* return_type = compilation()->method()->return_type();
 481     if (return_type->is_inlinetype()) {
 482       ciInlineKlass* vk = return_type->as_inline_klass();
 483       if (vk->can_be_returned_as_fields()) {
 484         address unpack_handler = vk->unpack_handler();
 485         assert(unpack_handler != nullptr, "must be");
 486         __ call(RuntimeAddress(unpack_handler));
 487       }
 488     } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
 489       Label skip;
 490       Label not_null;
 491       __ testptr(rax, rax);
 492       __ jcc(Assembler::notZero, not_null);
 493       // Returned value is null, zero all return registers because they may belong to oop fields
 494       __ xorq(j_rarg1, j_rarg1);
 495       __ xorq(j_rarg2, j_rarg2);
 496       __ xorq(j_rarg3, j_rarg3);
 497       __ xorq(j_rarg4, j_rarg4);
 498       __ xorq(j_rarg5, j_rarg5);
 499       __ jmp(skip);
 500       __ bind(not_null);
 501 
 502       // Check if we are returning an non-null inline type and load its fields into registers
 503       __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
 504 
 505       // Load fields from a buffered value with an inline class specific handler
 506       __ load_klass(rdi, rax, rscratch1);
 507       __ movptr(rdi, Address(rdi, InlineKlass::adr_members_offset()));
 508       __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
 509       // Unpack handler can be null if inline type is not scalarizable in returns
 510       __ testptr(rdi, rdi);
 511       __ jcc(Assembler::zero, skip);
 512       __ call(rdi);
 513 
 514       __ bind(skip);
 515     }
 516     // At this point, rax points to the value object (for interpreter or C1 caller).
 517     // The fields of the object are copied into registers (for C2 caller).
 518   }
 519 
 520   // Pop the stack before the safepoint code
 521   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 522 
 523   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 524     __ reserved_stack_check();
 525   }
 526 
 527   // Note: we do not need to round double result; float result has the right precision
 528   // the poll sets the condition code, but no data registers
 529 
 530   code_stub->set_safepoint_offset(__ offset());
 531   __ relocate(relocInfo::poll_return_type);
 532   __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
 533   __ ret(0);
 534 }
 535 
 536 
 537 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
 538   return (__ store_inline_type_fields_to_buf(vk, false));
 539 }
 540 
 541 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 542   guarantee(info != nullptr, "Shouldn't be null");
 543   int offset = __ offset();
 544   const Register poll_addr = rscratch1;
 545   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 546   add_debug_info_for_branch(info);
 547   __ relocate(relocInfo::poll_type);
 548   address pre_pc = __ pc();
 549   __ testl(rax, Address(poll_addr, 0));
 550   address post_pc = __ pc();
 551   guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
 552   return offset;
 553 }
 554 
 555 
 556 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 557   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 558 }
 559 
 560 void LIR_Assembler::swap_reg(Register a, Register b) {
 561   __ xchgptr(a, b);
 562 }
 563 
 564 
 565 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 566   assert(src->is_constant(), "should not call otherwise");
 567   assert(dest->is_register(), "should not call otherwise");
 568   LIR_Const* c = src->as_constant_ptr();
 569 
 570   switch (c->type()) {
 571     case T_INT: {
 572       assert(patch_code == lir_patch_none, "no patching handled here");
 573       __ movl(dest->as_register(), c->as_jint());
 574       break;
 575     }
 576 
 577     case T_ADDRESS: {
 578       assert(patch_code == lir_patch_none, "no patching handled here");
 579       __ movptr(dest->as_register(), c->as_jint());
 580       break;
 581     }
 582 
 583     case T_LONG: {
 584 #if INCLUDE_CDS
 585       if (AOTCodeCache::is_on_for_dump()) {
 586         address b = c->as_pointer();
 587         if (b == (address)ThreadIdentifier::unsafe_offset()) {
 588           __ lea(dest->as_register_lo(), ExternalAddress(b));
 589           break;
 590         }
 591       }
 592 #endif
 593       assert(patch_code == lir_patch_none, "no patching handled here");
 594 #if INCLUDE_CDS
 595       if (AOTCodeCache::is_on_for_dump()) {
 596         address b = c->as_pointer();
 597         if (b == (address)ThreadIdentifier::unsafe_offset()) {
 598           __ lea(dest->as_register_lo(), ExternalAddress(b));
 599           break;
 600         }
 601         if (AOTRuntimeConstants::contains(b)) {
 602           __ load_aotrc_address(dest->as_register_lo(), b);
 603           break;
 604         }
 605       }
 606 #endif
 607       __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
 608       break;
 609     }
 610 
 611     case T_OBJECT: {
 612       if (patch_code != lir_patch_none) {
 613         jobject2reg_with_patching(dest->as_register(), info);
 614       } else {
 615         __ movoop(dest->as_register(), c->as_jobject());
 616       }
 617       break;
 618     }
 619 
 620     case T_METADATA: {
 621       if (patch_code != lir_patch_none) {
 622         klass2reg_with_patching(dest->as_register(), info);
 623       } else {
 624         __ mov_metadata(dest->as_register(), c->as_metadata());
 625       }
 626       break;
 627     }
 628 
 629     case T_FLOAT: {
 630       if (dest->is_single_xmm()) {
 631         if (UseAVX <= 2 && c->is_zero_float()) {
 632           __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
 633         } else {
 634           __ movflt(dest->as_xmm_float_reg(),
 635                    InternalAddress(float_constant(c->as_jfloat())));
 636         }
 637       } else {
 638         ShouldNotReachHere();
 639       }
 640       break;
 641     }
 642 
 643     case T_DOUBLE: {
 644       if (dest->is_double_xmm()) {
 645         if (UseAVX <= 2 && c->is_zero_double()) {
 646           __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
 647         } else {
 648           __ movdbl(dest->as_xmm_double_reg(),
 649                     InternalAddress(double_constant(c->as_jdouble())));
 650         }
 651       } else {
 652         ShouldNotReachHere();
 653       }
 654       break;
 655     }
 656 
 657     default:
 658       ShouldNotReachHere();
 659   }
 660 }
 661 
 662 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 663   assert(src->is_constant(), "should not call otherwise");
 664   assert(dest->is_stack(), "should not call otherwise");
 665   LIR_Const* c = src->as_constant_ptr();
 666 
 667   switch (c->type()) {
 668     case T_INT:  // fall through
 669     case T_FLOAT:
 670       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 671       break;
 672 
 673     case T_ADDRESS:
 674       __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 675       break;
 676 
 677     case T_OBJECT:
 678       __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
 679       break;
 680 
 681     case T_LONG:  // fall through
 682     case T_DOUBLE:
 683       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 684                                               lo_word_offset_in_bytes),
 685                 (intptr_t)c->as_jlong_bits(),
 686                 rscratch1);
 687       break;
 688 
 689     default:
 690       ShouldNotReachHere();
 691   }
 692 }
 693 
 694 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 695   assert(src->is_constant(), "should not call otherwise");
 696   assert(dest->is_address(), "should not call otherwise");
 697   LIR_Const* c = src->as_constant_ptr();
 698   LIR_Address* addr = dest->as_address_ptr();
 699 
 700   int null_check_here = code_offset();
 701   switch (type) {
 702     case T_INT:    // fall through
 703     case T_FLOAT:
 704       __ movl(as_Address(addr), c->as_jint_bits());
 705       break;
 706 
 707     case T_ADDRESS:
 708       __ movptr(as_Address(addr), c->as_jint_bits());
 709       break;
 710 
 711     case T_OBJECT:  // fall through
 712     case T_ARRAY:
 713       if (c->as_jobject() == nullptr) {
 714         if (UseCompressedOops && !wide) {
 715           __ movl(as_Address(addr), NULL_WORD);
 716         } else {
 717           __ xorptr(rscratch1, rscratch1);
 718           null_check_here = code_offset();
 719           __ movptr(as_Address(addr), rscratch1);
 720         }
 721       } else {
 722         if (is_literal_address(addr)) {
 723           ShouldNotReachHere();
 724           __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
 725         } else {
 726           __ movoop(rscratch1, c->as_jobject());
 727           if (UseCompressedOops && !wide) {
 728             __ encode_heap_oop(rscratch1);
 729             null_check_here = code_offset();
 730             __ movl(as_Address_lo(addr), rscratch1);
 731           } else {
 732             null_check_here = code_offset();
 733             __ movptr(as_Address_lo(addr), rscratch1);
 734           }
 735         }
 736       }
 737       break;
 738 
 739     case T_LONG:    // fall through
 740     case T_DOUBLE:
 741       if (is_literal_address(addr)) {
 742         ShouldNotReachHere();
 743         __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
 744       } else {
 745         __ movptr(r10, (intptr_t)c->as_jlong_bits());
 746         null_check_here = code_offset();
 747         __ movptr(as_Address_lo(addr), r10);
 748       }
 749       break;
 750 
 751     case T_BOOLEAN: // fall through
 752     case T_BYTE:
 753       __ movb(as_Address(addr), c->as_jint() & 0xFF);
 754       break;
 755 
 756     case T_CHAR:    // fall through
 757     case T_SHORT:
 758       __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
 759       break;
 760 
 761     default:
 762       ShouldNotReachHere();
 763   };
 764 
 765   if (info != nullptr) {
 766     add_debug_info_for_null_check(null_check_here, info);
 767   }
 768 }
 769 
 770 
 771 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 772   assert(src->is_register(), "should not call otherwise");
 773   assert(dest->is_register(), "should not call otherwise");
 774 
 775   // move between cpu-registers
 776   if (dest->is_single_cpu()) {
 777     if (src->type() == T_LONG) {
 778       // Can do LONG -> OBJECT
 779       move_regs(src->as_register_lo(), dest->as_register());
 780       return;
 781     }
 782     assert(src->is_single_cpu(), "must match");
 783     if (src->type() == T_OBJECT) {
 784       __ verify_oop(src->as_register());
 785     }
 786     move_regs(src->as_register(), dest->as_register());
 787 
 788   } else if (dest->is_double_cpu()) {
 789     if (is_reference_type(src->type())) {
 790       // Surprising to me but we can see move of a long to t_object
 791       __ verify_oop(src->as_register());
 792       move_regs(src->as_register(), dest->as_register_lo());
 793       return;
 794     }
 795     assert(src->is_double_cpu(), "must match");
 796     Register f_lo = src->as_register_lo();
 797     Register f_hi = src->as_register_hi();
 798     Register t_lo = dest->as_register_lo();
 799     Register t_hi = dest->as_register_hi();
 800     assert(f_hi == f_lo, "must be same");
 801     assert(t_hi == t_lo, "must be same");
 802     move_regs(f_lo, t_lo);
 803 
 804     // move between xmm-registers
 805   } else if (dest->is_single_xmm()) {
 806     assert(src->is_single_xmm(), "must match");
 807     __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
 808   } else if (dest->is_double_xmm()) {
 809     assert(src->is_double_xmm(), "must match");
 810     __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
 811 
 812   } else {
 813     ShouldNotReachHere();
 814   }
 815 }
 816 
 817 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 818   assert(src->is_register(), "should not call otherwise");
 819   assert(dest->is_stack(), "should not call otherwise");
 820 
 821   if (src->is_single_cpu()) {
 822     Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
 823     if (is_reference_type(type)) {
 824       __ verify_oop(src->as_register());
 825       __ movptr (dst, src->as_register());
 826     } else if (type == T_METADATA || type == T_ADDRESS) {
 827       __ movptr (dst, src->as_register());
 828     } else {
 829       __ movl (dst, src->as_register());
 830     }
 831 
 832   } else if (src->is_double_cpu()) {
 833     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 834     Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
 835     __ movptr (dstLO, src->as_register_lo());
 836 
 837   } else if (src->is_single_xmm()) {
 838     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 839     __ movflt(dst_addr, src->as_xmm_float_reg());
 840 
 841   } else if (src->is_double_xmm()) {
 842     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 843     __ movdbl(dst_addr, src->as_xmm_double_reg());
 844 
 845   } else {
 846     ShouldNotReachHere();
 847   }
 848 }
 849 
 850 
 851 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 852   LIR_Address* to_addr = dest->as_address_ptr();
 853   PatchingStub* patch = nullptr;
 854   Register compressed_src = rscratch1;
 855 
 856   if (is_reference_type(type)) {
 857     __ verify_oop(src->as_register());
 858     if (UseCompressedOops && !wide) {
 859       __ movptr(compressed_src, src->as_register());
 860       __ encode_heap_oop(compressed_src);
 861       if (patch_code != lir_patch_none) {
 862         info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
 863       }
 864     }
 865   }
 866 
 867   if (patch_code != lir_patch_none) {
 868     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 869     Address toa = as_Address(to_addr);
 870     assert(toa.disp() != 0, "must have");
 871   }
 872 
 873   int null_check_here = code_offset();
 874   switch (type) {
 875     case T_FLOAT: {
 876       assert(src->is_single_xmm(), "not a float");
 877       __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
 878       break;
 879     }
 880 
 881     case T_DOUBLE: {
 882       assert(src->is_double_xmm(), "not a double");
 883       __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
 884       break;
 885     }
 886 
 887     case T_ARRAY:   // fall through
 888     case T_OBJECT:  // fall through
 889       if (UseCompressedOops && !wide) {
 890         __ movl(as_Address(to_addr), compressed_src);
 891       } else {
 892         __ movptr(as_Address(to_addr), src->as_register());
 893       }
 894       break;
 895     case T_ADDRESS:
 896       __ movptr(as_Address(to_addr), src->as_register());
 897       break;
 898     case T_INT:
 899       __ movl(as_Address(to_addr), src->as_register());
 900       break;
 901 
 902     case T_LONG: {
 903       Register from_lo = src->as_register_lo();
 904       Register from_hi = src->as_register_hi();
 905       __ movptr(as_Address_lo(to_addr), from_lo);
 906       break;
 907     }
 908 
 909     case T_BYTE:    // fall through
 910     case T_BOOLEAN: {
 911       Register src_reg = src->as_register();
 912       Address dst_addr = as_Address(to_addr);
 913       assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
 914       __ movb(dst_addr, src_reg);
 915       break;
 916     }
 917 
 918     case T_CHAR:    // fall through
 919     case T_SHORT:
 920       __ movw(as_Address(to_addr), src->as_register());
 921       break;
 922 
 923     default:
 924       ShouldNotReachHere();
 925   }
 926   if (info != nullptr) {
 927     add_debug_info_for_null_check(null_check_here, info);
 928   }
 929 
 930   if (patch_code != lir_patch_none) {
 931     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
 932   }
 933 }
 934 
 935 
 936 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 937   assert(src->is_stack(), "should not call otherwise");
 938   assert(dest->is_register(), "should not call otherwise");
 939 
 940   if (dest->is_single_cpu()) {
 941     if (is_reference_type(type)) {
 942       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 943       __ verify_oop(dest->as_register());
 944     } else if (type == T_METADATA || type == T_ADDRESS) {
 945       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 946     } else {
 947       __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 948     }
 949 
 950   } else if (dest->is_double_cpu()) {
 951     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 952     Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
 953     __ movptr(dest->as_register_lo(), src_addr_LO);
 954 
 955   } else if (dest->is_single_xmm()) {
 956     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 957     __ movflt(dest->as_xmm_float_reg(), src_addr);
 958 
 959   } else if (dest->is_double_xmm()) {
 960     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 961     __ movdbl(dest->as_xmm_double_reg(), src_addr);
 962 
 963   } else {
 964     ShouldNotReachHere();
 965   }
 966 }
 967 
 968 
 969 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 970   if (src->is_single_stack()) {
 971     if (is_reference_type(type)) {
 972       __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
 973       __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
 974     } else {
 975       //no pushl on 64bits
 976       __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
 977       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
 978     }
 979 
 980   } else if (src->is_double_stack()) {
 981     __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
 982     __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
 983 
 984   } else {
 985     ShouldNotReachHere();
 986   }
 987 }
 988 
 989 
 990 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 991   assert(src->is_address(), "should not call otherwise");
 992   assert(dest->is_register(), "should not call otherwise");
 993 
 994   LIR_Address* addr = src->as_address_ptr();
 995   Address from_addr = as_Address(addr);
 996 
 997   if (addr->base()->type() == T_OBJECT) {
 998     __ verify_oop(addr->base()->as_pointer_register());
 999   }
1000 
1001   switch (type) {
1002     case T_BOOLEAN: // fall through
1003     case T_BYTE:    // fall through
1004     case T_CHAR:    // fall through
1005     case T_SHORT:
1006       if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1007         // on pre P6 processors we may get partial register stalls
1008         // so blow away the value of to_rinfo before loading a
1009         // partial word into it.  Do it here so that it precedes
1010         // the potential patch point below.
1011         __ xorptr(dest->as_register(), dest->as_register());
1012       }
1013       break;
1014    default:
1015      break;
1016   }
1017 
1018   PatchingStub* patch = nullptr;
1019   if (patch_code != lir_patch_none) {
1020     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1021     assert(from_addr.disp() != 0, "must have");
1022   }
1023   if (info != nullptr) {
1024     add_debug_info_for_null_check_here(info);
1025   }
1026 
1027   switch (type) {
1028     case T_FLOAT: {
1029       if (dest->is_single_xmm()) {
1030         __ movflt(dest->as_xmm_float_reg(), from_addr);
1031       } else {
1032         ShouldNotReachHere();
1033       }
1034       break;
1035     }
1036 
1037     case T_DOUBLE: {
1038       if (dest->is_double_xmm()) {
1039         __ movdbl(dest->as_xmm_double_reg(), from_addr);
1040       } else {
1041         ShouldNotReachHere();
1042       }
1043       break;
1044     }
1045 
1046     case T_OBJECT:  // fall through
1047     case T_ARRAY:   // fall through
1048       if (UseCompressedOops && !wide) {
1049         __ movl(dest->as_register(), from_addr);
1050       } else {
1051         __ movptr(dest->as_register(), from_addr);
1052       }
1053       break;
1054 
1055     case T_ADDRESS:
1056       __ movptr(dest->as_register(), from_addr);
1057       break;
1058     case T_INT:
1059       __ movl(dest->as_register(), from_addr);
1060       break;
1061 
1062     case T_LONG: {
1063       Register to_lo = dest->as_register_lo();
1064       Register to_hi = dest->as_register_hi();
1065       __ movptr(to_lo, as_Address_lo(addr));
1066       break;
1067     }
1068 
1069     case T_BOOLEAN: // fall through
1070     case T_BYTE: {
1071       Register dest_reg = dest->as_register();
1072       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1073       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1074         __ movsbl(dest_reg, from_addr);
1075       } else {
1076         __ movb(dest_reg, from_addr);
1077         __ shll(dest_reg, 24);
1078         __ sarl(dest_reg, 24);
1079       }
1080       break;
1081     }
1082 
1083     case T_CHAR: {
1084       Register dest_reg = dest->as_register();
1085       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1086       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1087         __ movzwl(dest_reg, from_addr);
1088       } else {
1089         __ movw(dest_reg, from_addr);
1090       }
1091       break;
1092     }
1093 
1094     case T_SHORT: {
1095       Register dest_reg = dest->as_register();
1096       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1097         __ movswl(dest_reg, from_addr);
1098       } else {
1099         __ movw(dest_reg, from_addr);
1100         __ shll(dest_reg, 16);
1101         __ sarl(dest_reg, 16);
1102       }
1103       break;
1104     }
1105 
1106     default:
1107       ShouldNotReachHere();
1108   }
1109 
1110   if (patch != nullptr) {
1111     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1112   }
1113 
1114   if (is_reference_type(type)) {
1115     if (UseCompressedOops && !wide) {
1116       __ decode_heap_oop(dest->as_register());
1117     }
1118 
1119     __ verify_oop(dest->as_register());
1120   }
1121 }
1122 
1123 
1124 NEEDS_CLEANUP; // This could be static?
1125 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1126   int elem_size = type2aelembytes(type);
1127   switch (elem_size) {
1128     case 1: return Address::times_1;
1129     case 2: return Address::times_2;
1130     case 4: return Address::times_4;
1131     case 8: return Address::times_8;
1132   }
1133   ShouldNotReachHere();
1134   return Address::no_scale;
1135 }
1136 
1137 
1138 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1139   switch (op->code()) {
1140     case lir_idiv:
1141     case lir_irem:
1142       arithmetic_idiv(op->code(),
1143                       op->in_opr1(),
1144                       op->in_opr2(),
1145                       op->in_opr3(),
1146                       op->result_opr(),
1147                       op->info());
1148       break;
1149     case lir_fmad:
1150       __ fmad(op->result_opr()->as_xmm_double_reg(),
1151               op->in_opr1()->as_xmm_double_reg(),
1152               op->in_opr2()->as_xmm_double_reg(),
1153               op->in_opr3()->as_xmm_double_reg());
1154       break;
1155     case lir_fmaf:
1156       __ fmaf(op->result_opr()->as_xmm_float_reg(),
1157               op->in_opr1()->as_xmm_float_reg(),
1158               op->in_opr2()->as_xmm_float_reg(),
1159               op->in_opr3()->as_xmm_float_reg());
1160       break;
1161     default:      ShouldNotReachHere(); break;
1162   }
1163 }
1164 
1165 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1166 #ifdef ASSERT
1167   assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1168   if (op->block() != nullptr)  _branch_target_blocks.append(op->block());
1169   if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1170 #endif
1171 
1172   if (op->cond() == lir_cond_always) {
1173     if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1174     __ jmp (*(op->label()));
1175   } else {
1176     Assembler::Condition acond = Assembler::zero;
1177     if (op->code() == lir_cond_float_branch) {
1178       assert(op->ublock() != nullptr, "must have unordered successor");
1179       __ jcc(Assembler::parity, *(op->ublock()->label()));
1180       switch(op->cond()) {
1181         case lir_cond_equal:        acond = Assembler::equal;      break;
1182         case lir_cond_notEqual:     acond = Assembler::notEqual;   break;
1183         case lir_cond_less:         acond = Assembler::below;      break;
1184         case lir_cond_lessEqual:    acond = Assembler::belowEqual; break;
1185         case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1186         case lir_cond_greater:      acond = Assembler::above;      break;
1187         default:                         ShouldNotReachHere();
1188       }
1189     } else {
1190       switch (op->cond()) {
1191         case lir_cond_equal:        acond = Assembler::equal;       break;
1192         case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
1193         case lir_cond_less:         acond = Assembler::less;        break;
1194         case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
1195         case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1196         case lir_cond_greater:      acond = Assembler::greater;     break;
1197         case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
1198         case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
1199         default:                         ShouldNotReachHere();
1200       }
1201     }
1202     __ jcc(acond,*(op->label()));
1203   }
1204 }
1205 
1206 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1207   LIR_Opr src  = op->in_opr();
1208   LIR_Opr dest = op->result_opr();
1209 
1210   switch (op->bytecode()) {
1211     case Bytecodes::_i2l:
1212       __ movl2ptr(dest->as_register_lo(), src->as_register());
1213       break;
1214 
1215     case Bytecodes::_l2i:
1216       __ movl(dest->as_register(), src->as_register_lo());
1217       break;
1218 
1219     case Bytecodes::_i2b:
1220       move_regs(src->as_register(), dest->as_register());
1221       __ sign_extend_byte(dest->as_register());
1222       break;
1223 
1224     case Bytecodes::_i2c:
1225       move_regs(src->as_register(), dest->as_register());
1226       __ andl(dest->as_register(), 0xFFFF);
1227       break;
1228 
1229     case Bytecodes::_i2s:
1230       move_regs(src->as_register(), dest->as_register());
1231       __ sign_extend_short(dest->as_register());
1232       break;
1233 
1234     case Bytecodes::_f2d:
1235       __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1236       break;
1237 
1238     case Bytecodes::_d2f:
1239       __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1240       break;
1241 
1242     case Bytecodes::_i2f:
1243       __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1244       break;
1245 
1246     case Bytecodes::_i2d:
1247       __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1248       break;
1249 
1250     case Bytecodes::_l2f:
1251       __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1252       break;
1253 
1254     case Bytecodes::_l2d:
1255       __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1256       break;
1257 
1258     case Bytecodes::_f2i:
1259       __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1260       break;
1261 
1262     case Bytecodes::_d2i:
1263       __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1264       break;
1265 
1266     case Bytecodes::_f2l:
1267       __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1268       break;
1269 
1270     case Bytecodes::_d2l:
1271       __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1272       break;
1273 
1274     default: ShouldNotReachHere();
1275   }
1276 }
1277 
1278 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1279   if (op->init_check()) {
1280     add_debug_info_for_null_check_here(op->stub()->info());
1281     // init_state needs acquire, but x86 is TSO, and so we are already good.
1282     __ cmpb(Address(op->klass()->as_register(),
1283                     InstanceKlass::init_state_offset()),
1284                     InstanceKlass::fully_initialized);
1285     __ jcc(Assembler::notEqual, *op->stub()->entry());
1286   }
1287   __ allocate_object(op->obj()->as_register(),
1288                      op->tmp1()->as_register(),
1289                      op->tmp2()->as_register(),
1290                      op->header_size(),
1291                      op->object_size(),
1292                      op->klass()->as_register(),
1293                      *op->stub()->entry());
1294   __ bind(*op->stub()->continuation());
1295 }
1296 
1297 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1298   Register len =  op->len()->as_register();
1299   __ movslq(len, len);
1300 
1301   if (UseSlowPath || op->always_slow_path() ||
1302       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1303       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1304     __ jmp(*op->stub()->entry());
1305   } else {
1306     Register tmp1 = op->tmp1()->as_register();
1307     Register tmp2 = op->tmp2()->as_register();
1308     Register tmp3 = op->tmp3()->as_register();
1309     if (len == tmp1) {
1310       tmp1 = tmp3;
1311     } else if (len == tmp2) {
1312       tmp2 = tmp3;
1313     } else if (len == tmp3) {
1314       // everything is ok
1315     } else {
1316       __ mov(tmp3, len);
1317     }
1318     __ allocate_array(op->obj()->as_register(),
1319                       len,
1320                       tmp1,
1321                       tmp2,
1322                       arrayOopDesc::base_offset_in_bytes(op->type()),
1323                       array_element_size(op->type()),
1324                       op->klass()->as_register(),
1325                       *op->stub()->entry(),
1326                       op->zero_array());
1327   }
1328   __ bind(*op->stub()->continuation());
1329 }
1330 
1331 void LIR_Assembler::type_profile_helper(Register mdo,
1332                                         ciMethodData *md, ciProfileData *data,
1333                                         Register recv) {
1334   int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0));
1335   __ profile_receiver_type(recv, mdo, mdp_offset);
1336 }
1337 
1338 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1339   // we always need a stub for the failure case.
1340   CodeStub* stub = op->stub();
1341   Register obj = op->object()->as_register();
1342   Register k_RInfo = op->tmp1()->as_register();
1343   Register klass_RInfo = op->tmp2()->as_register();
1344   Register dst = op->result_opr()->as_register();
1345   ciKlass* k = op->klass();
1346   Register Rtmp1 = noreg;
1347   Register tmp_load_klass = rscratch1;
1348 
1349   // check if it needs to be profiled
1350   ciMethodData* md = nullptr;
1351   ciProfileData* data = nullptr;
1352 
1353   if (op->should_profile()) {
1354     ciMethod* method = op->profiled_method();
1355     assert(method != nullptr, "Should have method");
1356     int bci = op->profiled_bci();
1357     md = method->method_data_or_null();
1358     assert(md != nullptr, "Sanity");
1359     data = md->bci_to_data(bci);
1360     assert(data != nullptr,                "need data for type check");
1361     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1362   }
1363   Label* success_target = success;
1364   Label* failure_target = failure;
1365 
1366   if (obj == k_RInfo) {
1367     k_RInfo = dst;
1368   } else if (obj == klass_RInfo) {
1369     klass_RInfo = dst;
1370   }
1371   Rtmp1 = op->tmp3()->as_register();
1372   select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1373 
1374   assert_different_registers(obj, k_RInfo, klass_RInfo);
1375 
1376   if (op->need_null_check()) {
1377     __ testptr(obj, obj);
1378     if (op->should_profile()) {
1379       Label not_null;
1380       Register mdo  = klass_RInfo;
1381       __ mov_metadata(mdo, md->constant_encoding());
1382       __ jccb(Assembler::notEqual, not_null);
1383       // Object is null; update MDO and exit
1384       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1385       int header_bits = BitData::null_seen_byte_constant();
1386       __ orb(data_addr, header_bits);
1387       __ jmp(*obj_is_null);
1388       __ bind(not_null);
1389 
1390     Register recv = k_RInfo;
1391     __ load_klass(recv, obj, tmp_load_klass);
1392     type_profile_helper(mdo, md, data, recv);
1393     } else {
1394       __ jcc(Assembler::equal, *obj_is_null);
1395     }
1396   }
1397 
1398   if (!k->is_loaded()) {
1399     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1400   } else {
1401     __ mov_metadata(k_RInfo, k->constant_encoding());
1402   }
1403   __ verify_oop(obj);
1404 
1405   if (op->fast_check()) {
1406     assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1407     // get object class
1408     // not a safepoint as obj null check happens earlier
1409     __ load_klass(Rtmp1, obj, tmp_load_klass);
1410     __ cmpptr(k_RInfo, Rtmp1);
1411     __ jcc(Assembler::notEqual, *failure_target);
1412     // successful cast, fall through to profile or jump
1413   } else {
1414     // get object class
1415     // not a safepoint as obj null check happens earlier
1416     __ load_klass(klass_RInfo, obj, tmp_load_klass);
1417     if (k->is_loaded()) {
1418       // See if we get an immediate positive hit
1419       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1420       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1421         __ jcc(Assembler::notEqual, *failure_target);
1422         // successful cast, fall through to profile or jump
1423       } else {
1424         // See if we get an immediate positive hit
1425         __ jcc(Assembler::equal, *success_target);
1426         // check for self
1427         if (k->is_loaded() && k->is_obj_array_klass()) {
1428           // For a direct pointer comparison, we need the refined array klass pointer
1429           ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1430           __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1431           __ cmpptr(klass_RInfo, tmp_load_klass);
1432         } else {
1433           __ cmpptr(klass_RInfo, k_RInfo);
1434         }
1435         __ jcc(Assembler::equal, *success_target);
1436 
1437         __ push_ppx(klass_RInfo);
1438         __ push_ppx(k_RInfo);
1439         __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1440         __ pop_ppx(klass_RInfo);
1441         __ pop_ppx(klass_RInfo);
1442         // result is a boolean
1443         __ testl(klass_RInfo, klass_RInfo);
1444         __ jcc(Assembler::equal, *failure_target);
1445         // successful cast, fall through to profile or jump
1446       }
1447     } else {
1448       // perform the fast part of the checking logic
1449       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1450       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1451       __ push_ppx(klass_RInfo);
1452       __ push_ppx(k_RInfo);
1453       __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1454       __ pop_ppx(klass_RInfo);
1455       __ pop_ppx(k_RInfo);
1456       // result is a boolean
1457       __ testl(k_RInfo, k_RInfo);
1458       __ jcc(Assembler::equal, *failure_target);
1459       // successful cast, fall through to profile or jump
1460     }
1461   }
1462   __ jmp(*success);
1463 }
1464 
1465 
1466 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1467   Register tmp_load_klass = rscratch1;
1468   LIR_Code code = op->code();
1469   if (code == lir_store_check) {
1470     Register value = op->object()->as_register();
1471     Register array = op->array()->as_register();
1472     Register k_RInfo = op->tmp1()->as_register();
1473     Register klass_RInfo = op->tmp2()->as_register();
1474     Register Rtmp1 = op->tmp3()->as_register();
1475 
1476     CodeStub* stub = op->stub();
1477 
1478     // check if it needs to be profiled
1479     ciMethodData* md = nullptr;
1480     ciProfileData* data = nullptr;
1481 
1482     if (op->should_profile()) {
1483       ciMethod* method = op->profiled_method();
1484       assert(method != nullptr, "Should have method");
1485       int bci = op->profiled_bci();
1486       md = method->method_data_or_null();
1487       assert(md != nullptr, "Sanity");
1488       data = md->bci_to_data(bci);
1489       assert(data != nullptr,                "need data for type check");
1490       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1491     }
1492     Label done;
1493     Label* success_target = &done;
1494     Label* failure_target = stub->entry();
1495 
1496     __ testptr(value, value);
1497     if (op->should_profile()) {
1498       Label not_null;
1499       Register mdo  = klass_RInfo;
1500       __ mov_metadata(mdo, md->constant_encoding());
1501       __ jccb(Assembler::notEqual, not_null);
1502       // Object is null; update MDO and exit
1503       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1504       int header_bits = BitData::null_seen_byte_constant();
1505       __ orb(data_addr, header_bits);
1506       __ jmp(done);
1507       __ bind(not_null);
1508 
1509       Register recv = k_RInfo;
1510       __ load_klass(recv, value, tmp_load_klass);
1511       type_profile_helper(mdo, md, data, recv);
1512     } else {
1513       __ jcc(Assembler::equal, done);
1514     }
1515 
1516     add_debug_info_for_null_check_here(op->info_for_exception());
1517     __ load_klass(k_RInfo, array, tmp_load_klass);
1518     __ load_klass(klass_RInfo, value, tmp_load_klass);
1519 
1520     // get instance klass (it's already uncompressed)
1521     __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1522     // perform the fast part of the checking logic
1523     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1524     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1525     __ push_ppx(klass_RInfo);
1526     __ push_ppx(k_RInfo);
1527     __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1528     __ pop_ppx(klass_RInfo);
1529     __ pop_ppx(k_RInfo);
1530     // result is a boolean
1531     __ testl(k_RInfo, k_RInfo);
1532     __ jcc(Assembler::equal, *failure_target);
1533     // fall through to the success case
1534 
1535     __ bind(done);
1536   } else
1537     if (code == lir_checkcast) {
1538       Register obj = op->object()->as_register();
1539       Register dst = op->result_opr()->as_register();
1540       Label success;
1541       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1542       __ bind(success);
1543       if (dst != obj) {
1544         __ mov(dst, obj);
1545       }
1546     } else
1547       if (code == lir_instanceof) {
1548         Register obj = op->object()->as_register();
1549         Register dst = op->result_opr()->as_register();
1550         Label success, failure, done;
1551         emit_typecheck_helper(op, &success, &failure, &failure);
1552         __ bind(failure);
1553         __ xorptr(dst, dst);
1554         __ jmpb(done);
1555         __ bind(success);
1556         __ movptr(dst, 1);
1557         __ bind(done);
1558       } else {
1559         ShouldNotReachHere();
1560       }
1561 
1562 }
1563 
1564 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1565   // We are loading/storing from/to an array that *may* be a flat array (the
1566   // declared type is Object[], abstract[], interface[] or VT.ref[]).
1567   // If this array is a flat array, take the slow path.
1568   __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1569 }
1570 
1571 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1572   // We are storing into an array that *may* be null-free (the declared type is
1573   // Object[], abstract[], interface[] or VT.ref[]).
1574   Label test_mark_word;
1575   Register tmp = op->tmp()->as_register();
1576   __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1577   __ testl(tmp, markWord::unlocked_value);
1578   __ jccb(Assembler::notZero, test_mark_word);
1579   __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1580   __ bind(test_mark_word);
1581   __ testl(tmp, markWord::null_free_array_bit_in_place);
1582 }
1583 
1584 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1585   Label L_oops_equal;
1586   Label L_oops_not_equal;
1587   Label L_end;
1588 
1589   Register left  = op->left()->as_register();
1590   Register right = op->right()->as_register();
1591 
1592   __ cmpptr(left, right);
1593   __ jcc(Assembler::equal, L_oops_equal);
1594 
1595   // (1) Null check -- if one of the operands is null, the other must not be null (because
1596   //     the two references are not equal), so they are not substitutable,
1597   __ testptr(left, left);
1598   __ jcc(Assembler::zero, L_oops_not_equal);
1599   __ testptr(right, right);
1600   __ jcc(Assembler::zero, L_oops_not_equal);
1601 
1602   ciKlass* left_klass = op->left_klass();
1603   ciKlass* right_klass = op->right_klass();
1604 
1605   // (2) Inline type check -- if either of the operands is not a inline type,
1606   //     they are not substitutable. We do this only if we are not sure that the
1607   //     operands are inline type
1608   if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1609       !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1610     Register tmp = op->tmp1()->as_register();
1611     __ movptr(tmp, (intptr_t)markWord::inline_type_pattern);
1612     __ andptr(tmp, Address(left, oopDesc::mark_offset_in_bytes()));
1613     __ andptr(tmp, Address(right, oopDesc::mark_offset_in_bytes()));
1614     __ cmpptr(tmp, (intptr_t)markWord::inline_type_pattern);
1615     __ jcc(Assembler::notEqual, L_oops_not_equal);
1616   }
1617 
1618   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1619   if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1620     // No need to load klass -- the operands are statically known to be the same inline klass.
1621     __ jmp(*op->stub()->entry());
1622   } else {
1623     Register tmp1 = op->tmp1()->as_register();
1624     Register tmp2 = op->tmp2()->as_register();
1625     if (left == right) { // same operand, so clearly the same klasses, let's save the check
1626       __ jmp (*op->stub()->entry());  //  -> do slow check
1627     } else {
1628       __ cmp_klasses_from_objects(left, right, tmp1, tmp2);
1629       __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1630     }
1631     // fall through to L_oops_not_equal
1632   }
1633 
1634   __ bind(L_oops_not_equal);
1635   move(op->not_equal_result(), op->result_opr());
1636   __ jmp(L_end);
1637 
1638   __ bind(L_oops_equal);
1639   move(op->equal_result(), op->result_opr());
1640   __ jmp(L_end);
1641 
1642   // We've returned from the stub. RAX contains 0x0 IFF the two
1643   // operands are not substitutable. (Don't compare against 0x1 in case the
1644   // C compiler is naughty)
1645   __ bind(*op->stub()->continuation());
1646   __ cmpl(rax, 0);
1647   __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1648   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1649   // fall-through
1650   __ bind(L_end);
1651 }
1652 
1653 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1654   if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1655     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1656     Register newval = op->new_value()->as_register();
1657     Register cmpval = op->cmp_value()->as_register();
1658     assert(cmpval == rax, "wrong register");
1659     assert(newval != noreg, "new val must be register");
1660     assert(cmpval != newval, "cmp and new values must be in different registers");
1661     assert(cmpval != addr, "cmp and addr must be in different registers");
1662     assert(newval != addr, "new value and addr must be in different registers");
1663 
1664     if (op->code() == lir_cas_obj) {
1665       if (UseCompressedOops) {
1666         __ encode_heap_oop(cmpval);
1667         __ mov(rscratch1, newval);
1668         __ encode_heap_oop(rscratch1);
1669         __ lock();
1670         // cmpval (rax) is implicitly used by this instruction
1671         __ cmpxchgl(rscratch1, Address(addr, 0));
1672       } else {
1673         __ lock();
1674         __ cmpxchgptr(newval, Address(addr, 0));
1675       }
1676     } else {
1677       assert(op->code() == lir_cas_int, "lir_cas_int expected");
1678       __ lock();
1679       __ cmpxchgl(newval, Address(addr, 0));
1680     }
1681   } else if (op->code() == lir_cas_long) {
1682     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1683     Register newval = op->new_value()->as_register_lo();
1684     Register cmpval = op->cmp_value()->as_register_lo();
1685     assert(cmpval == rax, "wrong register");
1686     assert(newval != noreg, "new val must be register");
1687     assert(cmpval != newval, "cmp and new values must be in different registers");
1688     assert(cmpval != addr, "cmp and addr must be in different registers");
1689     assert(newval != addr, "new value and addr must be in different registers");
1690     __ lock();
1691     __ cmpxchgq(newval, Address(addr, 0));
1692   } else {
1693     Unimplemented();
1694   }
1695 }
1696 
1697 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1698   assert(dst->is_cpu_register(), "must be");
1699   assert(dst->type() == src->type(), "must be");
1700 
1701   if (src->is_cpu_register()) {
1702     reg2reg(src, dst);
1703   } else if (src->is_stack()) {
1704     stack2reg(src, dst, dst->type());
1705   } else if (src->is_constant()) {
1706     const2reg(src, dst, lir_patch_none, nullptr);
1707   } else {
1708     ShouldNotReachHere();
1709   }
1710 }
1711 
1712 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1713                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1714   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1715 
1716   Assembler::Condition acond, ncond;
1717   switch (condition) {
1718     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1719     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1720     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1721     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1722     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1723     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1724     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1725     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1726     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1727                                 ShouldNotReachHere();
1728   }
1729 
1730   if (opr1->is_cpu_register()) {
1731     reg2reg(opr1, result);
1732   } else if (opr1->is_stack()) {
1733     stack2reg(opr1, result, result->type());
1734   } else if (opr1->is_constant()) {
1735     const2reg(opr1, result, lir_patch_none, nullptr);
1736   } else {
1737     ShouldNotReachHere();
1738   }
1739 
1740   if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1741     // optimized version that does not require a branch
1742     if (opr2->is_single_cpu()) {
1743       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1744       __ cmov(ncond, result->as_register(), opr2->as_register());
1745     } else if (opr2->is_double_cpu()) {
1746       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1747       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1748       __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1749     } else if (opr2->is_single_stack()) {
1750       __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1751     } else if (opr2->is_double_stack()) {
1752       __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1753     } else {
1754       ShouldNotReachHere();
1755     }
1756 
1757   } else {
1758     Label skip;
1759     __ jccb(acond, skip);
1760     if (opr2->is_cpu_register()) {
1761       reg2reg(opr2, result);
1762     } else if (opr2->is_stack()) {
1763       stack2reg(opr2, result, result->type());
1764     } else if (opr2->is_constant()) {
1765       const2reg(opr2, result, lir_patch_none, nullptr);
1766     } else {
1767       ShouldNotReachHere();
1768     }
1769     __ bind(skip);
1770   }
1771 }
1772 
1773 
1774 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1775   assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1776 
1777   if (left->is_single_cpu()) {
1778     assert(left == dest, "left and dest must be equal");
1779     Register lreg = left->as_register();
1780 
1781     if (right->is_single_cpu()) {
1782       // cpu register - cpu register
1783       Register rreg = right->as_register();
1784       switch (code) {
1785         case lir_add: __ addl (lreg, rreg); break;
1786         case lir_sub: __ subl (lreg, rreg); break;
1787         case lir_mul: __ imull(lreg, rreg); break;
1788         default:      ShouldNotReachHere();
1789       }
1790 
1791     } else if (right->is_stack()) {
1792       // cpu register - stack
1793       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1794       switch (code) {
1795         case lir_add: __ addl(lreg, raddr); break;
1796         case lir_sub: __ subl(lreg, raddr); break;
1797         default:      ShouldNotReachHere();
1798       }
1799 
1800     } else if (right->is_constant()) {
1801       // cpu register - constant
1802       jint c = right->as_constant_ptr()->as_jint();
1803       switch (code) {
1804         case lir_add: {
1805           __ incrementl(lreg, c);
1806           break;
1807         }
1808         case lir_sub: {
1809           __ decrementl(lreg, c);
1810           break;
1811         }
1812         default: ShouldNotReachHere();
1813       }
1814 
1815     } else {
1816       ShouldNotReachHere();
1817     }
1818 
1819   } else if (left->is_double_cpu()) {
1820     assert(left == dest, "left and dest must be equal");
1821     Register lreg_lo = left->as_register_lo();
1822     Register lreg_hi = left->as_register_hi();
1823 
1824     if (right->is_double_cpu()) {
1825       // cpu register - cpu register
1826       Register rreg_lo = right->as_register_lo();
1827       Register rreg_hi = right->as_register_hi();
1828       assert_different_registers(lreg_lo, rreg_lo);
1829       switch (code) {
1830         case lir_add:
1831           __ addptr(lreg_lo, rreg_lo);
1832           break;
1833         case lir_sub:
1834           __ subptr(lreg_lo, rreg_lo);
1835           break;
1836         case lir_mul:
1837           __ imulq(lreg_lo, rreg_lo);
1838           break;
1839         default:
1840           ShouldNotReachHere();
1841       }
1842 
1843     } else if (right->is_constant()) {
1844       // cpu register - constant
1845       jlong c = right->as_constant_ptr()->as_jlong_bits();
1846       __ movptr(r10, (intptr_t) c);
1847       switch (code) {
1848         case lir_add:
1849           __ addptr(lreg_lo, r10);
1850           break;
1851         case lir_sub:
1852           __ subptr(lreg_lo, r10);
1853           break;
1854         default:
1855           ShouldNotReachHere();
1856       }
1857 
1858     } else {
1859       ShouldNotReachHere();
1860     }
1861 
1862   } else if (left->is_single_xmm()) {
1863     assert(left == dest, "left and dest must be equal");
1864     XMMRegister lreg = left->as_xmm_float_reg();
1865 
1866     if (right->is_single_xmm()) {
1867       XMMRegister rreg = right->as_xmm_float_reg();
1868       switch (code) {
1869         case lir_add: __ addss(lreg, rreg);  break;
1870         case lir_sub: __ subss(lreg, rreg);  break;
1871         case lir_mul: __ mulss(lreg, rreg);  break;
1872         case lir_div: __ divss(lreg, rreg);  break;
1873         default: ShouldNotReachHere();
1874       }
1875     } else {
1876       Address raddr;
1877       if (right->is_single_stack()) {
1878         raddr = frame_map()->address_for_slot(right->single_stack_ix());
1879       } else if (right->is_constant()) {
1880         // hack for now
1881         raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1882       } else {
1883         ShouldNotReachHere();
1884       }
1885       switch (code) {
1886         case lir_add: __ addss(lreg, raddr);  break;
1887         case lir_sub: __ subss(lreg, raddr);  break;
1888         case lir_mul: __ mulss(lreg, raddr);  break;
1889         case lir_div: __ divss(lreg, raddr);  break;
1890         default: ShouldNotReachHere();
1891       }
1892     }
1893 
1894   } else if (left->is_double_xmm()) {
1895     assert(left == dest, "left and dest must be equal");
1896 
1897     XMMRegister lreg = left->as_xmm_double_reg();
1898     if (right->is_double_xmm()) {
1899       XMMRegister rreg = right->as_xmm_double_reg();
1900       switch (code) {
1901         case lir_add: __ addsd(lreg, rreg);  break;
1902         case lir_sub: __ subsd(lreg, rreg);  break;
1903         case lir_mul: __ mulsd(lreg, rreg);  break;
1904         case lir_div: __ divsd(lreg, rreg);  break;
1905         default: ShouldNotReachHere();
1906       }
1907     } else {
1908       Address raddr;
1909       if (right->is_double_stack()) {
1910         raddr = frame_map()->address_for_slot(right->double_stack_ix());
1911       } else if (right->is_constant()) {
1912         // hack for now
1913         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1914       } else {
1915         ShouldNotReachHere();
1916       }
1917       switch (code) {
1918         case lir_add: __ addsd(lreg, raddr);  break;
1919         case lir_sub: __ subsd(lreg, raddr);  break;
1920         case lir_mul: __ mulsd(lreg, raddr);  break;
1921         case lir_div: __ divsd(lreg, raddr);  break;
1922         default: ShouldNotReachHere();
1923       }
1924     }
1925 
1926   } else if (left->is_single_stack() || left->is_address()) {
1927     assert(left == dest, "left and dest must be equal");
1928 
1929     Address laddr;
1930     if (left->is_single_stack()) {
1931       laddr = frame_map()->address_for_slot(left->single_stack_ix());
1932     } else if (left->is_address()) {
1933       laddr = as_Address(left->as_address_ptr());
1934     } else {
1935       ShouldNotReachHere();
1936     }
1937 
1938     if (right->is_single_cpu()) {
1939       Register rreg = right->as_register();
1940       switch (code) {
1941         case lir_add: __ addl(laddr, rreg); break;
1942         case lir_sub: __ subl(laddr, rreg); break;
1943         default:      ShouldNotReachHere();
1944       }
1945     } else if (right->is_constant()) {
1946       jint c = right->as_constant_ptr()->as_jint();
1947       switch (code) {
1948         case lir_add: {
1949           __ incrementl(laddr, c);
1950           break;
1951         }
1952         case lir_sub: {
1953           __ decrementl(laddr, c);
1954           break;
1955         }
1956         default: ShouldNotReachHere();
1957       }
1958     } else {
1959       ShouldNotReachHere();
1960     }
1961 
1962   } else {
1963     ShouldNotReachHere();
1964   }
1965 }
1966 
1967 
1968 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1969   if (value->is_double_xmm()) {
1970     switch(code) {
1971       case lir_abs :
1972         {
1973           if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1974             __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1975           }
1976           assert(!tmp->is_valid(), "do not need temporary");
1977           __ andpd(dest->as_xmm_double_reg(),
1978                    ExternalAddress((address)double_signmask_pool),
1979                    rscratch1);
1980         }
1981         break;
1982 
1983       case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1984       // all other intrinsics are not available in the SSE instruction set, so FPU is used
1985       default      : ShouldNotReachHere();
1986     }
1987 
1988   } else if (code == lir_f2hf) {
1989     __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
1990   } else if (code == lir_hf2f) {
1991     __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
1992   } else {
1993     Unimplemented();
1994   }
1995 }
1996 
1997 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1998   // assert(left->destroys_register(), "check");
1999   if (left->is_single_cpu()) {
2000     Register reg = left->as_register();
2001     if (right->is_constant()) {
2002       int val = right->as_constant_ptr()->as_jint();
2003       switch (code) {
2004         case lir_logic_and: __ andl (reg, val); break;
2005         case lir_logic_or:  __ orl  (reg, val); break;
2006         case lir_logic_xor: __ xorl (reg, val); break;
2007         default: ShouldNotReachHere();
2008       }
2009     } else if (right->is_stack()) {
2010       // added support for stack operands
2011       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2012       switch (code) {
2013         case lir_logic_and: __ andl (reg, raddr); break;
2014         case lir_logic_or:  __ orl  (reg, raddr); break;
2015         case lir_logic_xor: __ xorl (reg, raddr); break;
2016         default: ShouldNotReachHere();
2017       }
2018     } else {
2019       Register rright = right->as_register();
2020       switch (code) {
2021         case lir_logic_and: __ andptr (reg, rright); break;
2022         case lir_logic_or : __ orptr  (reg, rright); break;
2023         case lir_logic_xor: __ xorptr (reg, rright); break;
2024         default: ShouldNotReachHere();
2025       }
2026     }
2027     move_regs(reg, dst->as_register());
2028   } else {
2029     Register l_lo = left->as_register_lo();
2030     Register l_hi = left->as_register_hi();
2031     if (right->is_constant()) {
2032       __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2033       switch (code) {
2034         case lir_logic_and:
2035           __ andq(l_lo, rscratch1);
2036           break;
2037         case lir_logic_or:
2038           __ orq(l_lo, rscratch1);
2039           break;
2040         case lir_logic_xor:
2041           __ xorq(l_lo, rscratch1);
2042           break;
2043         default: ShouldNotReachHere();
2044       }
2045     } else {
2046       Register r_lo;
2047       if (is_reference_type(right->type())) {
2048         r_lo = right->as_register();
2049       } else {
2050         r_lo = right->as_register_lo();
2051       }
2052       switch (code) {
2053         case lir_logic_and:
2054           __ andptr(l_lo, r_lo);
2055           break;
2056         case lir_logic_or:
2057           __ orptr(l_lo, r_lo);
2058           break;
2059         case lir_logic_xor:
2060           __ xorptr(l_lo, r_lo);
2061           break;
2062         default: ShouldNotReachHere();
2063       }
2064     }
2065 
2066     Register dst_lo = dst->as_register_lo();
2067     Register dst_hi = dst->as_register_hi();
2068 
2069     move_regs(l_lo, dst_lo);
2070   }
2071 }
2072 
2073 
2074 // we assume that rax, and rdx can be overwritten
2075 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2076 
2077   assert(left->is_single_cpu(),   "left must be register");
2078   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2079   assert(result->is_single_cpu(), "result must be register");
2080 
2081   //  assert(left->destroys_register(), "check");
2082   //  assert(right->destroys_register(), "check");
2083 
2084   Register lreg = left->as_register();
2085   Register dreg = result->as_register();
2086 
2087   if (right->is_constant()) {
2088     jint divisor = right->as_constant_ptr()->as_jint();
2089     assert(divisor > 0 && is_power_of_2(divisor), "must be");
2090     if (code == lir_idiv) {
2091       assert(lreg == rax, "must be rax,");
2092       assert(temp->as_register() == rdx, "tmp register must be rdx");
2093       __ cdql(); // sign extend into rdx:rax
2094       if (divisor == 2) {
2095         __ subl(lreg, rdx);
2096       } else {
2097         __ andl(rdx, divisor - 1);
2098         __ addl(lreg, rdx);
2099       }
2100       __ sarl(lreg, log2i_exact(divisor));
2101       move_regs(lreg, dreg);
2102     } else if (code == lir_irem) {
2103       Label done;
2104       __ mov(dreg, lreg);
2105       __ andl(dreg, 0x80000000 | (divisor - 1));
2106       __ jcc(Assembler::positive, done);
2107       __ decrement(dreg);
2108       __ orl(dreg, ~(divisor - 1));
2109       __ increment(dreg);
2110       __ bind(done);
2111     } else {
2112       ShouldNotReachHere();
2113     }
2114   } else {
2115     Register rreg = right->as_register();
2116     assert(lreg == rax, "left register must be rax,");
2117     assert(rreg != rdx, "right register must not be rdx");
2118     assert(temp->as_register() == rdx, "tmp register must be rdx");
2119 
2120     move_regs(lreg, rax);
2121 
2122     int idivl_offset = __ corrected_idivl(rreg);
2123     if (ImplicitDiv0Checks) {
2124       add_debug_info_for_div0(idivl_offset, info);
2125     }
2126     if (code == lir_irem) {
2127       move_regs(rdx, dreg); // result is in rdx
2128     } else {
2129       move_regs(rax, dreg);
2130     }
2131   }
2132 }
2133 
2134 
2135 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2136   if (opr1->is_single_cpu()) {
2137     Register reg1 = opr1->as_register();
2138     if (opr2->is_single_cpu()) {
2139       // cpu register - cpu register
2140       if (is_reference_type(opr1->type())) {
2141         __ cmpoop(reg1, opr2->as_register());
2142       } else {
2143         assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2144         __ cmpl(reg1, opr2->as_register());
2145       }
2146     } else if (opr2->is_stack()) {
2147       // cpu register - stack
2148       if (is_reference_type(opr1->type())) {
2149         __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2150       } else {
2151         __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2152       }
2153     } else if (opr2->is_constant()) {
2154       // cpu register - constant
2155       LIR_Const* c = opr2->as_constant_ptr();
2156       if (c->type() == T_INT) {
2157         jint i = c->as_jint();
2158         if (i == 0) {
2159           __ testl(reg1, reg1);
2160         } else {
2161           __ cmpl(reg1, i);
2162         }
2163       } else if (c->type() == T_METADATA) {
2164         // All we need for now is a comparison with null for equality.
2165         assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2166         Metadata* m = c->as_metadata();
2167         if (m == nullptr) {
2168           __ testptr(reg1, reg1);
2169         } else {
2170           ShouldNotReachHere();
2171         }
2172       } else if (is_reference_type(c->type())) {
2173         // In 64bit oops are single register
2174         jobject o = c->as_jobject();
2175         if (o == nullptr) {
2176           __ testptr(reg1, reg1);
2177         } else {
2178           __ cmpoop(reg1, o, rscratch1);
2179         }
2180       } else {
2181         fatal("unexpected type: %s", basictype_to_str(c->type()));
2182       }
2183       // cpu register - address
2184     } else if (opr2->is_address()) {
2185       if (op->info() != nullptr) {
2186         add_debug_info_for_null_check_here(op->info());
2187       }
2188       __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2189     } else {
2190       ShouldNotReachHere();
2191     }
2192 
2193   } else if(opr1->is_double_cpu()) {
2194     Register xlo = opr1->as_register_lo();
2195     Register xhi = opr1->as_register_hi();
2196     if (opr2->is_double_cpu()) {
2197       __ cmpptr(xlo, opr2->as_register_lo());
2198     } else if (opr2->is_constant()) {
2199       // cpu register - constant 0
2200       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2201       __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2202     } else {
2203       ShouldNotReachHere();
2204     }
2205 
2206   } else if (opr1->is_single_xmm()) {
2207     XMMRegister reg1 = opr1->as_xmm_float_reg();
2208     if (opr2->is_single_xmm()) {
2209       // xmm register - xmm register
2210       __ ucomiss(reg1, opr2->as_xmm_float_reg());
2211     } else if (opr2->is_stack()) {
2212       // xmm register - stack
2213       __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2214     } else if (opr2->is_constant()) {
2215       // xmm register - constant
2216       __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2217     } else if (opr2->is_address()) {
2218       // xmm register - address
2219       if (op->info() != nullptr) {
2220         add_debug_info_for_null_check_here(op->info());
2221       }
2222       __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2223     } else {
2224       ShouldNotReachHere();
2225     }
2226 
2227   } else if (opr1->is_double_xmm()) {
2228     XMMRegister reg1 = opr1->as_xmm_double_reg();
2229     if (opr2->is_double_xmm()) {
2230       // xmm register - xmm register
2231       __ ucomisd(reg1, opr2->as_xmm_double_reg());
2232     } else if (opr2->is_stack()) {
2233       // xmm register - stack
2234       __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2235     } else if (opr2->is_constant()) {
2236       // xmm register - constant
2237       __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2238     } else if (opr2->is_address()) {
2239       // xmm register - address
2240       if (op->info() != nullptr) {
2241         add_debug_info_for_null_check_here(op->info());
2242       }
2243       __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2244     } else {
2245       ShouldNotReachHere();
2246     }
2247 
2248   } else if (opr1->is_address() && opr2->is_constant()) {
2249     LIR_Const* c = opr2->as_constant_ptr();
2250     if (is_reference_type(c->type())) {
2251       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2252       __ movoop(rscratch1, c->as_jobject());
2253     }
2254     if (op->info() != nullptr) {
2255       add_debug_info_for_null_check_here(op->info());
2256     }
2257     // special case: address - constant
2258     LIR_Address* addr = opr1->as_address_ptr();
2259     if (c->type() == T_INT) {
2260       __ cmpl(as_Address(addr), c->as_jint());
2261     } else if (is_reference_type(c->type())) {
2262       // %%% Make this explode if addr isn't reachable until we figure out a
2263       // better strategy by giving noreg as the temp for as_Address
2264       __ cmpoop(rscratch1, as_Address(addr, noreg));
2265     } else {
2266       ShouldNotReachHere();
2267     }
2268 
2269   } else {
2270     ShouldNotReachHere();
2271   }
2272 }
2273 
2274 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2275   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2276     if (left->is_single_xmm()) {
2277       assert(right->is_single_xmm(), "must match");
2278       __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2279     } else if (left->is_double_xmm()) {
2280       assert(right->is_double_xmm(), "must match");
2281       __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2282 
2283     } else {
2284       ShouldNotReachHere();
2285     }
2286   } else {
2287     assert(code == lir_cmp_l2i, "check");
2288     Label done;
2289     Register dest = dst->as_register();
2290     __ cmpptr(left->as_register_lo(), right->as_register_lo());
2291     __ movl(dest, -1);
2292     __ jccb(Assembler::less, done);
2293     __ setb(Assembler::notZero, dest);
2294     __ movzbl(dest, dest);
2295     __ bind(done);
2296   }
2297 }
2298 
2299 
2300 void LIR_Assembler::align_call(LIR_Code code) {
2301   // make sure that the displacement word of the call ends up word aligned
2302   int offset = __ offset();
2303   switch (code) {
2304   case lir_static_call:
2305   case lir_optvirtual_call:
2306   case lir_dynamic_call:
2307     offset += NativeCall::displacement_offset;
2308     break;
2309   case lir_icvirtual_call:
2310     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2311     break;
2312   default: ShouldNotReachHere();
2313   }
2314   __ align(BytesPerWord, offset);
2315 }
2316 
2317 
2318 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2319   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2320          "must be aligned");
2321   __ call(AddressLiteral(op->addr(), rtype));
2322   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2323   __ post_call_nop();
2324 }
2325 
2326 
2327 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2328   __ ic_call(op->addr());
2329   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2330   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2331          "must be aligned");
2332   __ post_call_nop();
2333 }
2334 
2335 
2336 void LIR_Assembler::emit_static_call_stub() {
2337   address call_pc = __ pc();
2338   address stub = __ start_a_stub(call_stub_size());
2339   if (stub == nullptr) {
2340     bailout("static call stub overflow");
2341     return;
2342   }
2343 
2344   int start = __ offset();
2345 
2346   // make sure that the displacement word of the call ends up word aligned
2347   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2348   __ relocate(static_stub_Relocation::spec(call_pc));
2349   __ mov_metadata(rbx, (Metadata*)nullptr);
2350   // must be set to -1 at code generation time
2351   assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2352   // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2353   __ jump(RuntimeAddress(__ pc()));
2354 
2355   assert(__ offset() - start <= call_stub_size(), "stub too big");
2356   __ end_a_stub();
2357 }
2358 
2359 
2360 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2361   assert(exceptionOop->as_register() == rax, "must match");
2362   assert(exceptionPC->as_register() == rdx, "must match");
2363 
2364   // exception object is not added to oop map by LinearScan
2365   // (LinearScan assumes that no oops are in fixed registers)
2366   info->add_register_oop(exceptionOop);
2367   StubId unwind_id;
2368 
2369   // get current pc information
2370   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2371   int pc_for_athrow_offset = __ offset();
2372   InternalAddress pc_for_athrow(__ pc());
2373   __ lea(exceptionPC->as_register(), pc_for_athrow);
2374   add_call_info(pc_for_athrow_offset, info); // for exception handler
2375 
2376   __ verify_not_null_oop(rax);
2377   // search an exception handler (rax: exception oop, rdx: throwing pc)
2378   if (compilation()->has_fpu_code()) {
2379     unwind_id = StubId::c1_handle_exception_id;
2380   } else {
2381     unwind_id = StubId::c1_handle_exception_nofpu_id;
2382   }
2383   __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2384 
2385   // enough room for two byte trap
2386   __ nop();
2387 }
2388 
2389 
2390 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2391   assert(exceptionOop->as_register() == rax, "must match");
2392 
2393   __ jmp(_unwind_handler_entry);
2394 }
2395 
2396 
2397 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2398 
2399   // optimized version for linear scan:
2400   // * count must be already in ECX (guaranteed by LinearScan)
2401   // * left and dest must be equal
2402   // * tmp must be unused
2403   assert(count->as_register() == SHIFT_count, "count must be in ECX");
2404   assert(left == dest, "left and dest must be equal");
2405   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2406 
2407   if (left->is_single_cpu()) {
2408     Register value = left->as_register();
2409     assert(value != SHIFT_count, "left cannot be ECX");
2410 
2411     switch (code) {
2412       case lir_shl:  __ shll(value); break;
2413       case lir_shr:  __ sarl(value); break;
2414       case lir_ushr: __ shrl(value); break;
2415       default: ShouldNotReachHere();
2416     }
2417   } else if (left->is_double_cpu()) {
2418     Register lo = left->as_register_lo();
2419     Register hi = left->as_register_hi();
2420     assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2421     switch (code) {
2422       case lir_shl:  __ shlptr(lo);        break;
2423       case lir_shr:  __ sarptr(lo);        break;
2424       case lir_ushr: __ shrptr(lo);        break;
2425       default: ShouldNotReachHere();
2426     }
2427   } else {
2428     ShouldNotReachHere();
2429   }
2430 }
2431 
2432 
2433 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2434   if (dest->is_single_cpu()) {
2435     // first move left into dest so that left is not destroyed by the shift
2436     Register value = dest->as_register();
2437     count = count & 0x1F; // Java spec
2438 
2439     move_regs(left->as_register(), value);
2440     switch (code) {
2441       case lir_shl:  __ shll(value, count); break;
2442       case lir_shr:  __ sarl(value, count); break;
2443       case lir_ushr: __ shrl(value, count); break;
2444       default: ShouldNotReachHere();
2445     }
2446   } else if (dest->is_double_cpu()) {
2447     // first move left into dest so that left is not destroyed by the shift
2448     Register value = dest->as_register_lo();
2449     count = count & 0x1F; // Java spec
2450 
2451     move_regs(left->as_register_lo(), value);
2452     switch (code) {
2453       case lir_shl:  __ shlptr(value, count); break;
2454       case lir_shr:  __ sarptr(value, count); break;
2455       case lir_ushr: __ shrptr(value, count); break;
2456       default: ShouldNotReachHere();
2457     }
2458   } else {
2459     ShouldNotReachHere();
2460   }
2461 }
2462 
2463 
2464 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2465   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2466   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2467   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2468   __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2469 }
2470 
2471 
2472 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2473   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2474   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2475   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2476   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2477 }
2478 
2479 
2480 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2481   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2482   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2483   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2484   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2485 }
2486 
2487 
2488 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2489   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2490   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2491   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2492   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2493 }
2494 
2495 
2496 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2497   if (null_check) {
2498     __ testptr(obj, obj);
2499     __ jcc(Assembler::zero, *slow_path->entry());
2500   }
2501   if (is_dest) {
2502     __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2503     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2504   } else {
2505     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2506   }
2507 }
2508 
2509 
2510 // This code replaces a call to arraycopy; no exception may
2511 // be thrown in this code, they must be thrown in the System.arraycopy
2512 // activation frame; we could save some checks if this would not be the case
2513 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2514   ciArrayKlass* default_type = op->expected_type();
2515   Register src = op->src()->as_register();
2516   Register dst = op->dst()->as_register();
2517   Register src_pos = op->src_pos()->as_register();
2518   Register dst_pos = op->dst_pos()->as_register();
2519   Register length  = op->length()->as_register();
2520   Register tmp = op->tmp()->as_register();
2521   Register tmp_load_klass = rscratch1;
2522   Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2523 
2524   CodeStub* stub = op->stub();
2525   int flags = op->flags();
2526   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2527   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2528 
2529   if (flags & LIR_OpArrayCopy::always_slow_path) {
2530     __ jmp(*stub->entry());
2531     __ bind(*stub->continuation());
2532     return;
2533   }
2534 
2535   // if we don't know anything, just go through the generic arraycopy
2536   if (default_type == nullptr) {
2537     // save outgoing arguments on stack in case call to System.arraycopy is needed
2538     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2539     // for interpreter calling conventions. Now we have to do it in new style conventions.
2540     // For the moment until C1 gets the new register allocator I just force all the
2541     // args to the right place (except the register args) and then on the back side
2542     // reload the register args properly if we go slow path. Yuck
2543 
2544     // These are proper for the calling convention
2545     store_parameter(length, 2);
2546     store_parameter(dst_pos, 1);
2547     store_parameter(dst, 0);
2548 
2549     // these are just temporary placements until we need to reload
2550     store_parameter(src_pos, 3);
2551     store_parameter(src, 4);
2552 
2553     address copyfunc_addr = StubRoutines::generic_arraycopy();
2554     assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2555 
2556     // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2557     // The arguments are in java calling convention so we can trivially shift them to C
2558     // convention
2559     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2560     __ mov(c_rarg0, j_rarg0);
2561     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2562     __ mov(c_rarg1, j_rarg1);
2563     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2564     __ mov(c_rarg2, j_rarg2);
2565     assert_different_registers(c_rarg3, j_rarg4);
2566     __ mov(c_rarg3, j_rarg3);
2567 #ifdef _WIN64
2568     // Allocate abi space for args but be sure to keep stack aligned
2569     __ subptr(rsp, 6*wordSize);
2570     store_parameter(j_rarg4, 4);
2571 #ifndef PRODUCT
2572     if (PrintC1Statistics) {
2573       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2574     }
2575 #endif
2576     __ call(RuntimeAddress(copyfunc_addr));
2577     __ addptr(rsp, 6*wordSize);
2578 #else
2579     __ mov(c_rarg4, j_rarg4);
2580 #ifndef PRODUCT
2581     if (PrintC1Statistics) {
2582       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2583     }
2584 #endif
2585     __ call(RuntimeAddress(copyfunc_addr));
2586 #endif // _WIN64
2587 
2588     __ testl(rax, rax);
2589     __ jcc(Assembler::equal, *stub->continuation());
2590 
2591     __ mov(tmp, rax);
2592     __ xorl(tmp, -1);
2593 
2594     // Reload values from the stack so they are where the stub
2595     // expects them.
2596     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2597     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2598     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2599     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2600     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2601 
2602     __ subl(length, tmp);
2603     __ addl(src_pos, tmp);
2604     __ addl(dst_pos, tmp);
2605     __ jmp(*stub->entry());
2606 
2607     __ bind(*stub->continuation());
2608     return;
2609   }
2610 
2611   // Handle inline type arrays
2612   if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2613     arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2614   }
2615   if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2616     arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2617   }
2618 
2619   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2620 
2621   int elem_size = type2aelembytes(basic_type);
2622   Address::ScaleFactor scale;
2623 
2624   switch (elem_size) {
2625     case 1 :
2626       scale = Address::times_1;
2627       break;
2628     case 2 :
2629       scale = Address::times_2;
2630       break;
2631     case 4 :
2632       scale = Address::times_4;
2633       break;
2634     case 8 :
2635       scale = Address::times_8;
2636       break;
2637     default:
2638       scale = Address::no_scale;
2639       ShouldNotReachHere();
2640   }
2641 
2642   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2643   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2644 
2645   // length and pos's are all sign extended at this point on 64bit
2646 
2647   // test for null
2648   if (flags & LIR_OpArrayCopy::src_null_check) {
2649     __ testptr(src, src);
2650     __ jcc(Assembler::zero, *stub->entry());
2651   }
2652   if (flags & LIR_OpArrayCopy::dst_null_check) {
2653     __ testptr(dst, dst);
2654     __ jcc(Assembler::zero, *stub->entry());
2655   }
2656 
2657   // If the compiler was not able to prove that exact type of the source or the destination
2658   // of the arraycopy is an array type, check at runtime if the source or the destination is
2659   // an instance type.
2660   if (flags & LIR_OpArrayCopy::type_check) {
2661     if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2662       __ load_klass(tmp, dst, tmp_load_klass);
2663       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2664       __ jcc(Assembler::greaterEqual, *stub->entry());
2665     }
2666 
2667     if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2668       __ load_klass(tmp, src, tmp_load_klass);
2669       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2670       __ jcc(Assembler::greaterEqual, *stub->entry());
2671     }
2672   }
2673 
2674   // check if negative
2675   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2676     __ testl(src_pos, src_pos);
2677     __ jcc(Assembler::less, *stub->entry());
2678   }
2679   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2680     __ testl(dst_pos, dst_pos);
2681     __ jcc(Assembler::less, *stub->entry());
2682   }
2683 
2684   if (flags & LIR_OpArrayCopy::src_range_check) {
2685     __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2686     __ cmpl(tmp, src_length_addr);
2687     __ jcc(Assembler::above, *stub->entry());
2688   }
2689   if (flags & LIR_OpArrayCopy::dst_range_check) {
2690     __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2691     __ cmpl(tmp, dst_length_addr);
2692     __ jcc(Assembler::above, *stub->entry());
2693   }
2694 
2695   if (flags & LIR_OpArrayCopy::length_positive_check) {
2696     __ testl(length, length);
2697     __ jcc(Assembler::less, *stub->entry());
2698   }
2699 
2700   __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2701   __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2702 
2703   if (flags & LIR_OpArrayCopy::type_check) {
2704     // We don't know the array types are compatible
2705     if (basic_type != T_OBJECT) {
2706       // Simple test for basic type arrays
2707       __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2708       __ jcc(Assembler::notEqual, *stub->entry());
2709     } else {
2710       // For object arrays, if src is a sub class of dst then we can
2711       // safely do the copy.
2712       Label cont, slow;
2713 
2714       __ push_ppx(src);
2715       __ push_ppx(dst);
2716 
2717       __ load_klass(src, src, tmp_load_klass);
2718       __ load_klass(dst, dst, tmp_load_klass);
2719 
2720       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2721 
2722       __ push_ppx(src);
2723       __ push_ppx(dst);
2724       __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2725       __ pop_ppx(dst);
2726       __ pop_ppx(src);
2727 
2728       __ testl(src, src);
2729       __ jcc(Assembler::notEqual, cont);
2730 
2731       __ bind(slow);
2732       __ pop_ppx(dst);
2733       __ pop_ppx(src);
2734 
2735       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2736       if (copyfunc_addr != nullptr) { // use stub if available
2737         // src is not a sub class of dst so we have to do a
2738         // per-element check.
2739 
2740         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2741         if ((flags & mask) != mask) {
2742           // Check that at least both of them object arrays.
2743           assert(flags & mask, "one of the two should be known to be an object array");
2744 
2745           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2746             __ load_klass(tmp, src, tmp_load_klass);
2747           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2748             __ load_klass(tmp, dst, tmp_load_klass);
2749           }
2750           int lh_offset = in_bytes(Klass::layout_helper_offset());
2751           Address klass_lh_addr(tmp, lh_offset);
2752           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2753           __ cmpl(klass_lh_addr, objArray_lh);
2754           __ jcc(Assembler::notEqual, *stub->entry());
2755         }
2756 
2757        // Spill because stubs can use any register they like and it's
2758        // easier to restore just those that we care about.
2759        store_parameter(dst, 0);
2760        store_parameter(dst_pos, 1);
2761        store_parameter(length, 2);
2762        store_parameter(src_pos, 3);
2763        store_parameter(src, 4);
2764 
2765         __ movl2ptr(length, length); //higher 32bits must be null
2766 
2767         __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2768         assert_different_registers(c_rarg0, dst, dst_pos, length);
2769         __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2770         assert_different_registers(c_rarg1, dst, length);
2771 
2772         __ mov(c_rarg2, length);
2773         assert_different_registers(c_rarg2, dst);
2774 
2775 #ifdef _WIN64
2776         // Allocate abi space for args but be sure to keep stack aligned
2777         __ subptr(rsp, 6*wordSize);
2778         __ load_klass(c_rarg3, dst, tmp_load_klass);
2779         __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2780         store_parameter(c_rarg3, 4);
2781         __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2782         __ call(RuntimeAddress(copyfunc_addr));
2783         __ addptr(rsp, 6*wordSize);
2784 #else
2785         __ load_klass(c_rarg4, dst, tmp_load_klass);
2786         __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2787         __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2788         __ call(RuntimeAddress(copyfunc_addr));
2789 #endif
2790 
2791 #ifndef PRODUCT
2792         if (PrintC1Statistics) {
2793           Label failed;
2794           __ testl(rax, rax);
2795           __ jcc(Assembler::notZero, failed);
2796           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2797           __ bind(failed);
2798         }
2799 #endif
2800 
2801         __ testl(rax, rax);
2802         __ jcc(Assembler::zero, *stub->continuation());
2803 
2804 #ifndef PRODUCT
2805         if (PrintC1Statistics) {
2806           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2807         }
2808 #endif
2809 
2810         __ mov(tmp, rax);
2811 
2812         __ xorl(tmp, -1);
2813 
2814         // Restore previously spilled arguments
2815         __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2816         __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2817         __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2818         __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2819         __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2820 
2821 
2822         __ subl(length, tmp);
2823         __ addl(src_pos, tmp);
2824         __ addl(dst_pos, tmp);
2825       }
2826 
2827       __ jmp(*stub->entry());
2828 
2829       __ bind(cont);
2830       __ pop(dst);
2831       __ pop(src);
2832     }
2833   }
2834 
2835 #ifdef ASSERT
2836   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2837     // Sanity check the known type with the incoming class.  For the
2838     // primitive case the types must match exactly with src.klass and
2839     // dst.klass each exactly matching the default type.  For the
2840     // object array case, if no type check is needed then either the
2841     // dst type is exactly the expected type and the src type is a
2842     // subtype which we can't check or src is the same array as dst
2843     // but not necessarily exactly of type default_type.
2844     Label known_ok, halt;
2845     __ mov_metadata(tmp, default_type->constant_encoding());
2846     __ encode_klass_not_null(tmp, rscratch1);
2847 
2848     if (basic_type != T_OBJECT) {
2849       __ cmp_klass(tmp, dst, tmp2);
2850       __ jcc(Assembler::notEqual, halt);
2851       __ cmp_klass(tmp, src, tmp2);
2852       __ jcc(Assembler::equal, known_ok);
2853     } else {
2854       __ cmp_klass(tmp, dst, tmp2);
2855       __ jcc(Assembler::equal, known_ok);
2856       __ cmpptr(src, dst);
2857       __ jcc(Assembler::equal, known_ok);
2858     }
2859     __ bind(halt);
2860     __ stop("incorrect type information in arraycopy");
2861     __ bind(known_ok);
2862   }
2863 #endif
2864 
2865 #ifndef PRODUCT
2866   if (PrintC1Statistics) {
2867     __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2868   }
2869 #endif
2870 
2871   assert_different_registers(c_rarg0, dst, dst_pos, length);
2872   __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2873   assert_different_registers(c_rarg1, length);
2874   __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2875   __ mov(c_rarg2, length);
2876 
2877   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2878   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2879   const char *name;
2880   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2881   __ call_VM_leaf(entry, 0);
2882 
2883   if (stub != nullptr) {
2884     __ bind(*stub->continuation());
2885   }
2886 }
2887 
2888 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2889   assert(op->crc()->is_single_cpu(),  "crc must be register");
2890   assert(op->val()->is_single_cpu(),  "byte value must be register");
2891   assert(op->result_opr()->is_single_cpu(), "result must be register");
2892   Register crc = op->crc()->as_register();
2893   Register val = op->val()->as_register();
2894   Register res = op->result_opr()->as_register();
2895 
2896   assert_different_registers(val, crc, res);
2897 
2898   __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2899   __ notl(crc); // ~crc
2900   __ update_byte_crc32(crc, val, res);
2901   __ notl(crc); // ~crc
2902   __ mov(res, crc);
2903 }
2904 
2905 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2906   Register obj = op->obj_opr()->as_register();  // may not be an oop
2907   Register hdr = op->hdr_opr()->as_register();
2908   Register lock = op->lock_opr()->as_register();
2909   if (op->code() == lir_lock) {
2910     Register tmp = op->scratch_opr()->as_register();
2911     // add debug info for NullPointerException only if one is possible
2912     int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2913     if (op->info() != nullptr) {
2914       add_debug_info_for_null_check(null_check_offset, op->info());
2915     }
2916     // done
2917   } else if (op->code() == lir_unlock) {
2918     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2919   } else {
2920     Unimplemented();
2921   }
2922   __ bind(*op->stub()->continuation());
2923 }
2924 
2925 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2926   Register obj = op->obj()->as_pointer_register();
2927   Register result = op->result_opr()->as_pointer_register();
2928 
2929   CodeEmitInfo* info = op->info();
2930   if (info != nullptr) {
2931     add_debug_info_for_null_check_here(info);
2932   }
2933 
2934   __ load_klass(result, obj, rscratch1);
2935 }
2936 
2937 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2938   ciMethod* method = op->profiled_method();
2939   int bci          = op->profiled_bci();
2940   ciMethod* callee = op->profiled_callee();
2941   Register tmp_load_klass = rscratch1;
2942 
2943   // Update counter for all call types
2944   ciMethodData* md = method->method_data_or_null();
2945   assert(md != nullptr, "Sanity");
2946   ciProfileData* data = md->bci_to_data(bci);
2947   assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2948   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2949   Register mdo  = op->mdo()->as_register();
2950   __ mov_metadata(mdo, md->constant_encoding());
2951   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2952   // Perform additional virtual call profiling for invokevirtual and
2953   // invokeinterface bytecodes
2954   if (op->should_profile_receiver_type()) {
2955     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2956     Register recv = op->recv()->as_register();
2957     assert_different_registers(mdo, recv);
2958     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2959     ciKlass* known_klass = op->known_holder();
2960     if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2961       // We know the type that will be seen at this call site; we can
2962       // statically update the MethodData* rather than needing to do
2963       // dynamic tests on the receiver type.
2964       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2965       for (uint i = 0; i < VirtualCallData::row_limit(); i++) {
2966         ciKlass* receiver = vc_data->receiver(i);
2967         if (known_klass->equals(receiver)) {
2968           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2969           __ addptr(data_addr, DataLayout::counter_increment);
2970           return;
2971         }
2972       }
2973       // Receiver type is not found in profile data.
2974       // Fall back to runtime helper to handle the rest at runtime.
2975       __ mov_metadata(recv, known_klass->constant_encoding());
2976     } else {
2977       __ load_klass(recv, recv, tmp_load_klass);
2978     }
2979     type_profile_helper(mdo, md, data, recv);
2980   } else {
2981     // Static call
2982     __ addptr(counter_addr, DataLayout::counter_increment);
2983   }
2984 }
2985 
2986 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2987   Register obj = op->obj()->as_register();
2988   Register tmp = op->tmp()->as_pointer_register();
2989   Register tmp_load_klass = rscratch1;
2990   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2991   ciKlass* exact_klass = op->exact_klass();
2992   intptr_t current_klass = op->current_klass();
2993   bool not_null = op->not_null();
2994   bool no_conflict = op->no_conflict();
2995 
2996   Label update, next, none;
2997 
2998   bool do_null = !not_null;
2999   bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3000   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3001 
3002   assert(do_null || do_update, "why are we here?");
3003   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3004 
3005   __ verify_oop(obj);
3006 
3007 #ifdef ASSERT
3008   if (obj == tmp) {
3009     assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3010   } else {
3011     assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3012   }
3013 #endif
3014   if (do_null) {
3015     __ testptr(obj, obj);
3016     __ jccb(Assembler::notZero, update);
3017     if (!TypeEntries::was_null_seen(current_klass)) {
3018       __ testptr(mdo_addr, TypeEntries::null_seen);
3019 #ifndef ASSERT
3020       __ jccb(Assembler::notZero, next); // already set
3021 #else
3022       __ jcc(Assembler::notZero, next); // already set
3023 #endif
3024       // atomic update to prevent overwriting Klass* with 0
3025       __ lock();
3026       __ orptr(mdo_addr, TypeEntries::null_seen);
3027     }
3028     if (do_update) {
3029 #ifndef ASSERT
3030       __ jmpb(next);
3031     }
3032 #else
3033       __ jmp(next);
3034     }
3035   } else {
3036     __ testptr(obj, obj);
3037     __ jcc(Assembler::notZero, update);
3038     __ stop("unexpected null obj");
3039 #endif
3040   }
3041 
3042   __ bind(update);
3043 
3044   if (do_update) {
3045 #ifdef ASSERT
3046     if (exact_klass != nullptr) {
3047       Label ok;
3048       __ load_klass(tmp, obj, tmp_load_klass);
3049       __ push_ppx(tmp);
3050       __ mov_metadata(tmp, exact_klass->constant_encoding());
3051       __ cmpptr(tmp, Address(rsp, 0));
3052       __ jcc(Assembler::equal, ok);
3053       __ stop("exact klass and actual klass differ");
3054       __ bind(ok);
3055       __ pop_ppx(tmp);
3056     }
3057 #endif
3058     if (!no_conflict) {
3059       if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3060         if (exact_klass != nullptr) {
3061           __ mov_metadata(tmp, exact_klass->constant_encoding());
3062         } else {
3063           __ load_klass(tmp, obj, tmp_load_klass);
3064         }
3065         __ mov(rscratch1, tmp); // save original value before XOR
3066         __ xorptr(tmp, mdo_addr);
3067         __ testptr(tmp, TypeEntries::type_klass_mask);
3068         // klass seen before, nothing to do. The unknown bit may have been
3069         // set already but no need to check.
3070         __ jccb(Assembler::zero, next);
3071 
3072         __ testptr(tmp, TypeEntries::type_unknown);
3073         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3074 
3075         if (TypeEntries::is_type_none(current_klass)) {
3076           __ testptr(mdo_addr, TypeEntries::type_mask);
3077           __ jccb(Assembler::zero, none);
3078           // There is a chance that the checks above (re-reading profiling
3079           // data from memory) fail if another thread has just set the
3080           // profiling to this obj's klass
3081           __ mov(tmp, rscratch1); // get back original value before XOR
3082           __ xorptr(tmp, mdo_addr);
3083           __ testptr(tmp, TypeEntries::type_klass_mask);
3084           __ jccb(Assembler::zero, next);
3085         }
3086       } else {
3087         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3088                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3089 
3090         __ testptr(mdo_addr, TypeEntries::type_unknown);
3091         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3092       }
3093 
3094       // different than before. Cannot keep accurate profile.
3095       __ orptr(mdo_addr, TypeEntries::type_unknown);
3096 
3097       if (TypeEntries::is_type_none(current_klass)) {
3098         __ jmpb(next);
3099 
3100         __ bind(none);
3101         // first time here. Set profile type.
3102         __ movptr(mdo_addr, tmp);
3103 #ifdef ASSERT
3104         __ andptr(tmp, TypeEntries::type_klass_mask);
3105         __ verify_klass_ptr(tmp);
3106 #endif
3107       }
3108     } else {
3109       // There's a single possible klass at this profile point
3110       assert(exact_klass != nullptr, "should be");
3111       if (TypeEntries::is_type_none(current_klass)) {
3112         __ mov_metadata(tmp, exact_klass->constant_encoding());
3113         __ xorptr(tmp, mdo_addr);
3114         __ testptr(tmp, TypeEntries::type_klass_mask);
3115 #ifdef ASSERT
3116         __ jcc(Assembler::zero, next);
3117 
3118         {
3119           Label ok;
3120           __ push_ppx(tmp);
3121           __ testptr(mdo_addr, TypeEntries::type_mask);
3122           __ jcc(Assembler::zero, ok);
3123           // may have been set by another thread
3124           __ mov_metadata(tmp, exact_klass->constant_encoding());
3125           __ xorptr(tmp, mdo_addr);
3126           __ testptr(tmp, TypeEntries::type_mask);
3127           __ jcc(Assembler::zero, ok);
3128 
3129           __ stop("unexpected profiling mismatch");
3130           __ bind(ok);
3131           __ pop_ppx(tmp);
3132         }
3133 #else
3134         __ jccb(Assembler::zero, next);
3135 #endif
3136         // first time here. Set profile type.
3137         __ movptr(mdo_addr, tmp);
3138 #ifdef ASSERT
3139         __ andptr(tmp, TypeEntries::type_klass_mask);
3140         __ verify_klass_ptr(tmp);
3141 #endif
3142       } else {
3143         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3144                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3145 
3146         __ testptr(mdo_addr, TypeEntries::type_unknown);
3147         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3148 
3149         __ orptr(mdo_addr, TypeEntries::type_unknown);
3150       }
3151     }
3152   }
3153   __ bind(next);
3154 }
3155 
3156 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3157   Register obj = op->obj()->as_register();
3158   Register tmp = op->tmp()->as_pointer_register();
3159   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3160   bool not_null = op->not_null();
3161   int flag = op->flag();
3162 
3163   Label not_inline_type;
3164   if (!not_null) {
3165     __ testptr(obj, obj);
3166     __ jccb(Assembler::zero, not_inline_type);
3167   }
3168 
3169   __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3170 
3171   __ orb(mdo_addr, flag);
3172 
3173   __ bind(not_inline_type);
3174 }
3175 
3176 
3177 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3178   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3179 }
3180 
3181 
3182 void LIR_Assembler::align_backward_branch_target() {
3183   __ align(BytesPerWord);
3184 }
3185 
3186 
3187 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3188   if (left->is_single_cpu()) {
3189     __ negl(left->as_register());
3190     move_regs(left->as_register(), dest->as_register());
3191 
3192   } else if (left->is_double_cpu()) {
3193     Register lo = left->as_register_lo();
3194     Register dst = dest->as_register_lo();
3195     __ movptr(dst, lo);
3196     __ negptr(dst);
3197 
3198   } else if (dest->is_single_xmm()) {
3199     assert(!tmp->is_valid(), "do not need temporary");
3200     if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3201       __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3202     }
3203     __ xorps(dest->as_xmm_float_reg(),
3204              ExternalAddress((address)float_signflip_pool),
3205              rscratch1);
3206   } else if (dest->is_double_xmm()) {
3207     assert(!tmp->is_valid(), "do not need temporary");
3208     if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3209       __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3210     }
3211     __ xorpd(dest->as_xmm_double_reg(),
3212              ExternalAddress((address)double_signflip_pool),
3213              rscratch1);
3214   } else {
3215     ShouldNotReachHere();
3216   }
3217 }
3218 
3219 
3220 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3221   assert(src->is_address(), "must be an address");
3222   assert(dest->is_register(), "must be a register");
3223 
3224   PatchingStub* patch = nullptr;
3225   if (patch_code != lir_patch_none) {
3226     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3227   }
3228 
3229   Register reg = dest->as_pointer_register();
3230   LIR_Address* addr = src->as_address_ptr();
3231   __ lea(reg, as_Address(addr));
3232 
3233   if (patch != nullptr) {
3234     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3235   }
3236 }
3237 
3238 
3239 
3240 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3241   assert(!tmp->is_valid(), "don't need temporary");
3242   __ call(RuntimeAddress(dest));
3243   if (info != nullptr) {
3244     add_call_info_here(info);
3245   }
3246   __ post_call_nop();
3247 }
3248 
3249 
3250 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3251   assert(type == T_LONG, "only for volatile long fields");
3252 
3253   if (info != nullptr) {
3254     add_debug_info_for_null_check_here(info);
3255   }
3256 
3257   if (src->is_double_xmm()) {
3258     if (dest->is_double_cpu()) {
3259       __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3260     } else if (dest->is_double_stack()) {
3261       __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3262     } else if (dest->is_address()) {
3263       __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3264     } else {
3265       ShouldNotReachHere();
3266     }
3267 
3268   } else if (dest->is_double_xmm()) {
3269     if (src->is_double_stack()) {
3270       __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3271     } else if (src->is_address()) {
3272       __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3273     } else {
3274       ShouldNotReachHere();
3275     }
3276 
3277   } else {
3278     ShouldNotReachHere();
3279   }
3280 }
3281 
3282 #ifdef ASSERT
3283 // emit run-time assertion
3284 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3285   assert(op->code() == lir_assert, "must be");
3286 
3287   if (op->in_opr1()->is_valid()) {
3288     assert(op->in_opr2()->is_valid(), "both operands must be valid");
3289     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3290   } else {
3291     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3292     assert(op->condition() == lir_cond_always, "no other conditions allowed");
3293   }
3294 
3295   Label ok;
3296   if (op->condition() != lir_cond_always) {
3297     Assembler::Condition acond = Assembler::zero;
3298     switch (op->condition()) {
3299       case lir_cond_equal:        acond = Assembler::equal;       break;
3300       case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
3301       case lir_cond_less:         acond = Assembler::less;        break;
3302       case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
3303       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3304       case lir_cond_greater:      acond = Assembler::greater;     break;
3305       case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
3306       case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
3307       default:                    ShouldNotReachHere();
3308     }
3309     __ jcc(acond, ok);
3310   }
3311   if (op->halt()) {
3312     const char* str = __ code_string(op->msg());
3313     __ stop(str);
3314   } else {
3315     breakpoint();
3316   }
3317   __ bind(ok);
3318 }
3319 #endif
3320 
3321 void LIR_Assembler::membar() {
3322   // QQQ sparc TSO uses this,
3323   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3324 }
3325 
3326 void LIR_Assembler::membar_acquire() {
3327   // No x86 machines currently require load fences
3328 }
3329 
3330 void LIR_Assembler::membar_release() {
3331   // No x86 machines currently require store fences
3332 }
3333 
3334 void LIR_Assembler::membar_loadload() {
3335   // no-op
3336   //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3337 }
3338 
3339 void LIR_Assembler::membar_storestore() {
3340   // no-op
3341   //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3342 }
3343 
3344 void LIR_Assembler::membar_loadstore() {
3345   // no-op
3346   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3347 }
3348 
3349 void LIR_Assembler::membar_storeload() {
3350   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3351 }
3352 
3353 void LIR_Assembler::on_spin_wait() {
3354   __ pause ();
3355 }
3356 
3357 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3358   assert(result_reg->is_register(), "check");
3359   __ mov(result_reg->as_register(), r15_thread);
3360 }
3361 
3362 void LIR_Assembler::check_orig_pc() {
3363   __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3364 }
3365 
3366 void LIR_Assembler::peephole(LIR_List*) {
3367   // do nothing for now
3368 }
3369 
3370 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3371   assert(data == dest, "xchg/xadd uses only 2 operands");
3372 
3373   if (data->type() == T_INT) {
3374     if (code == lir_xadd) {
3375       __ lock();
3376       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3377     } else {
3378       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3379     }
3380   } else if (data->is_oop()) {
3381     assert (code == lir_xchg, "xadd for oops");
3382     Register obj = data->as_register();
3383     if (UseCompressedOops) {
3384       __ encode_heap_oop(obj);
3385       __ xchgl(obj, as_Address(src->as_address_ptr()));
3386       __ decode_heap_oop(obj);
3387     } else {
3388       __ xchgptr(obj, as_Address(src->as_address_ptr()));
3389     }
3390   } else if (data->type() == T_LONG) {
3391     assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3392     if (code == lir_xadd) {
3393       __ lock();
3394       __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3395     } else {
3396       __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3397     }
3398   } else {
3399     ShouldNotReachHere();
3400   }
3401 }
3402 
3403 #undef __
--- EOF ---