1 /*
   2  * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInlineKlass.hpp"
  35 #include "ci/ciInstance.hpp"
  36 #include "ci/ciObjArrayKlass.hpp"
  37 #include "compiler/oopMap.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "gc/shared/gc_globals.hpp"
  40 #include "nativeInst_x86.hpp"
  41 #include "oops/oop.inline.hpp"
  42 #include "oops/objArrayKlass.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/safepointMechanism.hpp"
  45 #include "runtime/sharedRuntime.hpp"
  46 #include "runtime/stubRoutines.hpp"
  47 #include "utilities/powerOfTwo.hpp"
  48 #include "vmreg_x86.inline.hpp"
  49 
  50 
  51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  52 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  53 // fast versions of NegF/NegD and AbsF/AbsD.
  54 
  55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  57   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  58   // of 128-bits operands for SSE instructions.
  59   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  60   // Store the value to a 128-bits operand.
  61   operand[0] = lo;
  62   operand[1] = hi;
  63   return operand;
  64 }
  65 
  66 // Buffer for 128-bits masks used by SSE instructions.
  67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
  68 
  69 // Static initialization during VM startup.
  70 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2],         CONST64(0x7FFFFFFF7FFFFFFF),         CONST64(0x7FFFFFFF7FFFFFFF));
  71 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2],         CONST64(0x7FFFFFFFFFFFFFFF),         CONST64(0x7FFFFFFFFFFFFFFF));
  72 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
  73 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
  74 
  75 
  76 NEEDS_CLEANUP // remove this definitions ?
  77 const Register SYNC_header = rax;   // synchronization header
  78 const Register SHIFT_count = rcx;   // where count for shift operations must be
  79 
  80 #define __ _masm->
  81 
  82 
  83 static void select_different_registers(Register preserve,
  84                                        Register extra,
  85                                        Register &tmp1,
  86                                        Register &tmp2) {
  87   if (tmp1 == preserve) {
  88     assert_different_registers(tmp1, tmp2, extra);
  89     tmp1 = extra;
  90   } else if (tmp2 == preserve) {
  91     assert_different_registers(tmp1, tmp2, extra);
  92     tmp2 = extra;
  93   }
  94   assert_different_registers(preserve, tmp1, tmp2);
  95 }
  96 
  97 
  98 
  99 static void select_different_registers(Register preserve,
 100                                        Register extra,
 101                                        Register &tmp1,
 102                                        Register &tmp2,
 103                                        Register &tmp3) {
 104   if (tmp1 == preserve) {
 105     assert_different_registers(tmp1, tmp2, tmp3, extra);
 106     tmp1 = extra;
 107   } else if (tmp2 == preserve) {
 108     assert_different_registers(tmp1, tmp2, tmp3, extra);
 109     tmp2 = extra;
 110   } else if (tmp3 == preserve) {
 111     assert_different_registers(tmp1, tmp2, tmp3, extra);
 112     tmp3 = extra;
 113   }
 114   assert_different_registers(preserve, tmp1, tmp2, tmp3);
 115 }
 116 
 117 
 118 
 119 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
 120   if (opr->is_constant()) {
 121     LIR_Const* constant = opr->as_constant_ptr();
 122     switch (constant->type()) {
 123       case T_INT: {
 124         return true;
 125       }
 126 
 127       default:
 128         return false;
 129     }
 130   }
 131   return false;
 132 }
 133 
 134 
 135 LIR_Opr LIR_Assembler::receiverOpr() {
 136   return FrameMap::receiver_opr;
 137 }
 138 
 139 LIR_Opr LIR_Assembler::osrBufferPointer() {
 140   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 141 }
 142 
 143 //--------------fpu register translations-----------------------
 144 
 145 
 146 address LIR_Assembler::float_constant(float f) {
 147   address const_addr = __ float_constant(f);
 148   if (const_addr == nullptr) {
 149     bailout("const section overflow");
 150     return __ code()->consts()->start();
 151   } else {
 152     return const_addr;
 153   }
 154 }
 155 
 156 
 157 address LIR_Assembler::double_constant(double d) {
 158   address const_addr = __ double_constant(d);
 159   if (const_addr == nullptr) {
 160     bailout("const section overflow");
 161     return __ code()->consts()->start();
 162   } else {
 163     return const_addr;
 164   }
 165 }
 166 
 167 void LIR_Assembler::breakpoint() {
 168   __ int3();
 169 }
 170 
 171 void LIR_Assembler::push(LIR_Opr opr) {
 172   if (opr->is_single_cpu()) {
 173     __ push_reg(opr->as_register());
 174   } else if (opr->is_double_cpu()) {
 175     __ push_reg(opr->as_register_lo());
 176   } else if (opr->is_stack()) {
 177     __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
 178   } else if (opr->is_constant()) {
 179     LIR_Const* const_opr = opr->as_constant_ptr();
 180     if (const_opr->type() == T_OBJECT) {
 181       __ push_oop(const_opr->as_jobject(), rscratch1);
 182     } else if (const_opr->type() == T_INT) {
 183       __ push_jint(const_opr->as_jint());
 184     } else {
 185       ShouldNotReachHere();
 186     }
 187 
 188   } else {
 189     ShouldNotReachHere();
 190   }
 191 }
 192 
 193 void LIR_Assembler::pop(LIR_Opr opr) {
 194   if (opr->is_single_cpu()) {
 195     __ pop_reg(opr->as_register());
 196   } else {
 197     ShouldNotReachHere();
 198   }
 199 }
 200 
 201 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
 202   return addr->base()->is_illegal() && addr->index()->is_illegal();
 203 }
 204 
 205 //-------------------------------------------
 206 
 207 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 208   return as_Address(addr, rscratch1);
 209 }
 210 
 211 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 212   if (addr->base()->is_illegal()) {
 213     assert(addr->index()->is_illegal(), "must be illegal too");
 214     AddressLiteral laddr((address)addr->disp(), relocInfo::none);
 215     if (! __ reachable(laddr)) {
 216       __ movptr(tmp, laddr.addr());
 217       Address res(tmp, 0);
 218       return res;
 219     } else {
 220       return __ as_Address(laddr);
 221     }
 222   }
 223 
 224   Register base = addr->base()->as_pointer_register();
 225 
 226   if (addr->index()->is_illegal()) {
 227     return Address( base, addr->disp());
 228   } else if (addr->index()->is_cpu_register()) {
 229     Register index = addr->index()->as_pointer_register();
 230     return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
 231   } else if (addr->index()->is_constant()) {
 232     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 233     assert(Assembler::is_simm32(addr_offset), "must be");
 234 
 235     return Address(base, addr_offset);
 236   } else {
 237     Unimplemented();
 238     return Address();
 239   }
 240 }
 241 
 242 
 243 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 244   Address base = as_Address(addr);
 245   return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
 246 }
 247 
 248 
 249 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 250   return as_Address(addr);
 251 }
 252 
 253 
 254 void LIR_Assembler::osr_entry() {
 255   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 256   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 257   ValueStack* entry_state = osr_entry->state();
 258   int number_of_locks = entry_state->locks_size();
 259 
 260   // we jump here if osr happens with the interpreter
 261   // state set up to continue at the beginning of the
 262   // loop that triggered osr - in particular, we have
 263   // the following registers setup:
 264   //
 265   // rcx: osr buffer
 266   //
 267 
 268   // build frame
 269   ciMethod* m = compilation()->method();
 270   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 271 
 272   // OSR buffer is
 273   //
 274   // locals[nlocals-1..0]
 275   // monitors[0..number_of_locks]
 276   //
 277   // locals is a direct copy of the interpreter frame so in the osr buffer
 278   // so first slot in the local array is the last local from the interpreter
 279   // and last slot is local[0] (receiver) from the interpreter
 280   //
 281   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 282   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 283   // in the interpreter frame (the method lock if a sync method)
 284 
 285   // Initialize monitors in the compiled activation.
 286   //   rcx: pointer to osr buffer
 287   //
 288   // All other registers are dead at this point and the locals will be
 289   // copied into place by code emitted in the IR.
 290 
 291   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 292   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 293     int monitor_offset = BytesPerWord * method()->max_locals() +
 294       (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
 295     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 296     // the OSR buffer using 2 word entries: first the lock and then
 297     // the oop.
 298     for (int i = 0; i < number_of_locks; i++) {
 299       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 300 #ifdef ASSERT
 301       // verify the interpreter's monitor has a non-null object
 302       {
 303         Label L;
 304         __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
 305         __ jcc(Assembler::notZero, L);
 306         __ stop("locked object is null");
 307         __ bind(L);
 308       }
 309 #endif
 310       __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
 311       __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
 312       __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 313       __ movptr(frame_map()->address_for_monitor_object(i), rbx);
 314     }
 315   }
 316 }
 317 
 318 
 319 // inline cache check; done before the frame is built.
 320 int LIR_Assembler::check_icache() {
 321   return __ ic_check(CodeEntryAlignment);
 322 }
 323 
 324 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 325   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 326   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 327 
 328   Label L_skip_barrier;
 329   Register klass = rscratch1;
 330 
 331   __ mov_metadata(klass, method->holder()->constant_encoding());
 332   __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
 333 
 334   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 335 
 336   __ bind(L_skip_barrier);
 337 }
 338 
 339 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
 340   jobject o = nullptr;
 341   PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
 342   __ movoop(reg, o);
 343   patching_epilog(patch, lir_patch_normal, reg, info);
 344 }
 345 
 346 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 347   Metadata* o = nullptr;
 348   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 349   __ mov_metadata(reg, o);
 350   patching_epilog(patch, lir_patch_normal, reg, info);
 351 }
 352 
 353 // This specifies the rsp decrement needed to build the frame
 354 int LIR_Assembler::initial_frame_size_in_bytes() const {
 355   // if rounding, must let FrameMap know!
 356 
 357   // The frame_map records size in slots (32bit word)
 358 
 359   // subtract two words to account for return address and link
 360   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 361 }
 362 
 363 
 364 int LIR_Assembler::emit_exception_handler() {
 365   // generate code for exception handler
 366   address handler_base = __ start_a_stub(exception_handler_size());
 367   if (handler_base == nullptr) {
 368     // not enough space left for the handler
 369     bailout("exception handler overflow");
 370     return -1;
 371   }
 372 
 373   int offset = code_offset();
 374 
 375   // the exception oop and pc are in rax, and rdx
 376   // no other registers need to be preserved, so invalidate them
 377   __ invalidate_registers(false, true, true, false, true, true);
 378 
 379   // check that there is really an exception
 380   __ verify_not_null_oop(rax);
 381 
 382   // search an exception handler (rax: exception oop, rdx: throwing pc)
 383   __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id)));
 384   __ should_not_reach_here();
 385   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 386   __ end_a_stub();
 387 
 388   return offset;
 389 }
 390 
 391 
 392 // Emit the code to remove the frame from the stack in the exception
 393 // unwind path.
 394 int LIR_Assembler::emit_unwind_handler() {
 395 #ifndef PRODUCT
 396   if (CommentedAssembly) {
 397     _masm->block_comment("Unwind handler");
 398   }
 399 #endif
 400 
 401   int offset = code_offset();
 402 
 403   // Fetch the exception from TLS and clear out exception related thread state
 404   __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
 405   __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
 406   __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
 407 
 408   __ bind(_unwind_handler_entry);
 409   __ verify_not_null_oop(rax);
 410   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 411     __ mov(rbx, rax);  // Preserve the exception (rbx is always callee-saved)
 412   }
 413 
 414   // Perform needed unlocking
 415   MonitorExitStub* stub = nullptr;
 416   if (method()->is_synchronized()) {
 417     monitor_address(0, FrameMap::rax_opr);
 418     stub = new MonitorExitStub(FrameMap::rax_opr, 0);
 419     __ unlock_object(rdi, rsi, rax, *stub->entry());
 420     __ bind(*stub->continuation());
 421   }
 422 
 423   if (compilation()->env()->dtrace_method_probes()) {
 424     __ mov(rdi, r15_thread);
 425     __ mov_metadata(rsi, method()->constant_encoding());
 426     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 427   }
 428 
 429   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 430     __ mov(rax, rbx);  // Restore the exception
 431   }
 432 
 433   // remove the activation and dispatch to the unwind handler
 434   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 435   __ jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id)));
 436 
 437   // Emit the slow path assembly
 438   if (stub != nullptr) {
 439     stub->emit_code(this);
 440   }
 441 
 442   return offset;
 443 }
 444 
 445 
 446 int LIR_Assembler::emit_deopt_handler() {
 447   // generate code for exception handler
 448   address handler_base = __ start_a_stub(deopt_handler_size());
 449   if (handler_base == nullptr) {
 450     // not enough space left for the handler
 451     bailout("deopt handler overflow");
 452     return -1;
 453   }
 454 
 455   int offset = code_offset();
 456   InternalAddress here(__ pc());
 457 
 458   __ pushptr(here.addr(), rscratch1);
 459   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 460   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 461   __ end_a_stub();
 462 
 463   return offset;
 464 }
 465 
 466 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 467   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 468   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 469     assert(result->fpu() == 0, "result must already be on TOS");
 470   }
 471   if (InlineTypeReturnedAsFields) {
 472   #ifndef _LP64
 473      Unimplemented();
 474   #endif
 475     // Check if we are returning an non-null inline type and load its fields into registers
 476     ciType* return_type = compilation()->method()->return_type();
 477     if (return_type->is_inlinetype()) {
 478       ciInlineKlass* vk = return_type->as_inline_klass();
 479       if (vk->can_be_returned_as_fields()) {
 480         address unpack_handler = vk->unpack_handler();
 481         assert(unpack_handler != nullptr, "must be");
 482         __ call(RuntimeAddress(unpack_handler));
 483       }
 484     } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
 485       Label skip;
 486       Label not_null;
 487       __ testptr(rax, rax);
 488       __ jcc(Assembler::notZero, not_null);
 489       // Returned value is null, zero all return registers because they may belong to oop fields
 490       __ xorq(j_rarg1, j_rarg1);
 491       __ xorq(j_rarg2, j_rarg2);
 492       __ xorq(j_rarg3, j_rarg3);
 493       __ xorq(j_rarg4, j_rarg4);
 494       __ xorq(j_rarg5, j_rarg5);
 495       __ jmp(skip);
 496       __ bind(not_null);
 497 
 498       // Check if we are returning an non-null inline type and load its fields into registers
 499       __ test_oop_is_not_inline_type(rax, rscratch1, skip, /* can_be_null= */ false);
 500 
 501       // Load fields from a buffered value with an inline class specific handler
 502       __ load_klass(rdi, rax, rscratch1);
 503       __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 504       __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
 505       // Unpack handler can be null if inline type is not scalarizable in returns
 506       __ testptr(rdi, rdi);
 507       __ jcc(Assembler::zero, skip);
 508       __ call(rdi);
 509 
 510       __ bind(skip);
 511     }
 512     // At this point, rax points to the value object (for interpreter or C1 caller).
 513     // The fields of the object are copied into registers (for C2 caller).
 514   }
 515 
 516   // Pop the stack before the safepoint code
 517   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 518 
 519   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 520     __ reserved_stack_check();
 521   }
 522 
 523   // Note: we do not need to round double result; float result has the right precision
 524   // the poll sets the condition code, but no data registers
 525 
 526   code_stub->set_safepoint_offset(__ offset());
 527   __ relocate(relocInfo::poll_return_type);
 528   __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
 529   __ ret(0);
 530 }
 531 
 532 
 533 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
 534   return (__ store_inline_type_fields_to_buf(vk, false));
 535 }
 536 
 537 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 538   guarantee(info != nullptr, "Shouldn't be null");
 539   int offset = __ offset();
 540   const Register poll_addr = rscratch1;
 541   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 542   add_debug_info_for_branch(info);
 543   __ relocate(relocInfo::poll_type);
 544   address pre_pc = __ pc();
 545   __ testl(rax, Address(poll_addr, 0));
 546   address post_pc = __ pc();
 547   guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
 548   return offset;
 549 }
 550 
 551 
 552 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 553   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 554 }
 555 
 556 void LIR_Assembler::swap_reg(Register a, Register b) {
 557   __ xchgptr(a, b);
 558 }
 559 
 560 
 561 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 562   assert(src->is_constant(), "should not call otherwise");
 563   assert(dest->is_register(), "should not call otherwise");
 564   LIR_Const* c = src->as_constant_ptr();
 565 
 566   switch (c->type()) {
 567     case T_INT: {
 568       assert(patch_code == lir_patch_none, "no patching handled here");
 569       __ movl(dest->as_register(), c->as_jint());
 570       break;
 571     }
 572 
 573     case T_ADDRESS: {
 574       assert(patch_code == lir_patch_none, "no patching handled here");
 575       __ movptr(dest->as_register(), c->as_jint());
 576       break;
 577     }
 578 
 579     case T_LONG: {
 580       assert(patch_code == lir_patch_none, "no patching handled here");
 581       __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
 582       break;
 583     }
 584 
 585     case T_OBJECT: {
 586       if (patch_code != lir_patch_none) {
 587         jobject2reg_with_patching(dest->as_register(), info);
 588       } else {
 589         __ movoop(dest->as_register(), c->as_jobject());
 590       }
 591       break;
 592     }
 593 
 594     case T_METADATA: {
 595       if (patch_code != lir_patch_none) {
 596         klass2reg_with_patching(dest->as_register(), info);
 597       } else {
 598         __ mov_metadata(dest->as_register(), c->as_metadata());
 599       }
 600       break;
 601     }
 602 
 603     case T_FLOAT: {
 604       if (dest->is_single_xmm()) {
 605         if (UseAVX <= 2 && c->is_zero_float()) {
 606           __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
 607         } else {
 608           __ movflt(dest->as_xmm_float_reg(),
 609                    InternalAddress(float_constant(c->as_jfloat())));
 610         }
 611       } else {
 612         ShouldNotReachHere();
 613       }
 614       break;
 615     }
 616 
 617     case T_DOUBLE: {
 618       if (dest->is_double_xmm()) {
 619         if (UseAVX <= 2 && c->is_zero_double()) {
 620           __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
 621         } else {
 622           __ movdbl(dest->as_xmm_double_reg(),
 623                     InternalAddress(double_constant(c->as_jdouble())));
 624         }
 625       } else {
 626         ShouldNotReachHere();
 627       }
 628       break;
 629     }
 630 
 631     default:
 632       ShouldNotReachHere();
 633   }
 634 }
 635 
 636 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 637   assert(src->is_constant(), "should not call otherwise");
 638   assert(dest->is_stack(), "should not call otherwise");
 639   LIR_Const* c = src->as_constant_ptr();
 640 
 641   switch (c->type()) {
 642     case T_INT:  // fall through
 643     case T_FLOAT:
 644       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 645       break;
 646 
 647     case T_ADDRESS:
 648       __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 649       break;
 650 
 651     case T_OBJECT:
 652       __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
 653       break;
 654 
 655     case T_LONG:  // fall through
 656     case T_DOUBLE:
 657       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 658                                               lo_word_offset_in_bytes),
 659                 (intptr_t)c->as_jlong_bits(),
 660                 rscratch1);
 661       break;
 662 
 663     default:
 664       ShouldNotReachHere();
 665   }
 666 }
 667 
 668 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 669   assert(src->is_constant(), "should not call otherwise");
 670   assert(dest->is_address(), "should not call otherwise");
 671   LIR_Const* c = src->as_constant_ptr();
 672   LIR_Address* addr = dest->as_address_ptr();
 673 
 674   int null_check_here = code_offset();
 675   switch (type) {
 676     case T_INT:    // fall through
 677     case T_FLOAT:
 678       __ movl(as_Address(addr), c->as_jint_bits());
 679       break;
 680 
 681     case T_ADDRESS:
 682       __ movptr(as_Address(addr), c->as_jint_bits());
 683       break;
 684 
 685     case T_OBJECT:  // fall through
 686     case T_ARRAY:
 687       if (c->as_jobject() == nullptr) {
 688         if (UseCompressedOops && !wide) {
 689           __ movl(as_Address(addr), NULL_WORD);
 690         } else {
 691           __ xorptr(rscratch1, rscratch1);
 692           null_check_here = code_offset();
 693           __ movptr(as_Address(addr), rscratch1);
 694         }
 695       } else {
 696         if (is_literal_address(addr)) {
 697           ShouldNotReachHere();
 698           __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
 699         } else {
 700           __ movoop(rscratch1, c->as_jobject());
 701           if (UseCompressedOops && !wide) {
 702             __ encode_heap_oop(rscratch1);
 703             null_check_here = code_offset();
 704             __ movl(as_Address_lo(addr), rscratch1);
 705           } else {
 706             null_check_here = code_offset();
 707             __ movptr(as_Address_lo(addr), rscratch1);
 708           }
 709         }
 710       }
 711       break;
 712 
 713     case T_LONG:    // fall through
 714     case T_DOUBLE:
 715       if (is_literal_address(addr)) {
 716         ShouldNotReachHere();
 717         __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
 718       } else {
 719         __ movptr(r10, (intptr_t)c->as_jlong_bits());
 720         null_check_here = code_offset();
 721         __ movptr(as_Address_lo(addr), r10);
 722       }
 723       break;
 724 
 725     case T_BOOLEAN: // fall through
 726     case T_BYTE:
 727       __ movb(as_Address(addr), c->as_jint() & 0xFF);
 728       break;
 729 
 730     case T_CHAR:    // fall through
 731     case T_SHORT:
 732       __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
 733       break;
 734 
 735     default:
 736       ShouldNotReachHere();
 737   };
 738 
 739   if (info != nullptr) {
 740     add_debug_info_for_null_check(null_check_here, info);
 741   }
 742 }
 743 
 744 
 745 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 746   assert(src->is_register(), "should not call otherwise");
 747   assert(dest->is_register(), "should not call otherwise");
 748 
 749   // move between cpu-registers
 750   if (dest->is_single_cpu()) {
 751     if (src->type() == T_LONG) {
 752       // Can do LONG -> OBJECT
 753       move_regs(src->as_register_lo(), dest->as_register());
 754       return;
 755     }
 756     assert(src->is_single_cpu(), "must match");
 757     if (src->type() == T_OBJECT) {
 758       __ verify_oop(src->as_register());
 759     }
 760     move_regs(src->as_register(), dest->as_register());
 761 
 762   } else if (dest->is_double_cpu()) {
 763     if (is_reference_type(src->type())) {
 764       // Surprising to me but we can see move of a long to t_object
 765       __ verify_oop(src->as_register());
 766       move_regs(src->as_register(), dest->as_register_lo());
 767       return;
 768     }
 769     assert(src->is_double_cpu(), "must match");
 770     Register f_lo = src->as_register_lo();
 771     Register f_hi = src->as_register_hi();
 772     Register t_lo = dest->as_register_lo();
 773     Register t_hi = dest->as_register_hi();
 774     assert(f_hi == f_lo, "must be same");
 775     assert(t_hi == t_lo, "must be same");
 776     move_regs(f_lo, t_lo);
 777 
 778     // move between xmm-registers
 779   } else if (dest->is_single_xmm()) {
 780     assert(src->is_single_xmm(), "must match");
 781     __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
 782   } else if (dest->is_double_xmm()) {
 783     assert(src->is_double_xmm(), "must match");
 784     __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
 785 
 786   } else {
 787     ShouldNotReachHere();
 788   }
 789 }
 790 
 791 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 792   assert(src->is_register(), "should not call otherwise");
 793   assert(dest->is_stack(), "should not call otherwise");
 794 
 795   if (src->is_single_cpu()) {
 796     Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
 797     if (is_reference_type(type)) {
 798       __ verify_oop(src->as_register());
 799       __ movptr (dst, src->as_register());
 800     } else if (type == T_METADATA || type == T_ADDRESS) {
 801       __ movptr (dst, src->as_register());
 802     } else {
 803       __ movl (dst, src->as_register());
 804     }
 805 
 806   } else if (src->is_double_cpu()) {
 807     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 808     Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
 809     __ movptr (dstLO, src->as_register_lo());
 810 
 811   } else if (src->is_single_xmm()) {
 812     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 813     __ movflt(dst_addr, src->as_xmm_float_reg());
 814 
 815   } else if (src->is_double_xmm()) {
 816     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 817     __ movdbl(dst_addr, src->as_xmm_double_reg());
 818 
 819   } else {
 820     ShouldNotReachHere();
 821   }
 822 }
 823 
 824 
 825 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 826   LIR_Address* to_addr = dest->as_address_ptr();
 827   PatchingStub* patch = nullptr;
 828   Register compressed_src = rscratch1;
 829 
 830   if (is_reference_type(type)) {
 831     __ verify_oop(src->as_register());
 832     if (UseCompressedOops && !wide) {
 833       __ movptr(compressed_src, src->as_register());
 834       __ encode_heap_oop(compressed_src);
 835       if (patch_code != lir_patch_none) {
 836         info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
 837       }
 838     }
 839   }
 840 
 841   if (patch_code != lir_patch_none) {
 842     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 843     Address toa = as_Address(to_addr);
 844     assert(toa.disp() != 0, "must have");
 845   }
 846 
 847   int null_check_here = code_offset();
 848   switch (type) {
 849     case T_FLOAT: {
 850       assert(src->is_single_xmm(), "not a float");
 851       __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
 852       break;
 853     }
 854 
 855     case T_DOUBLE: {
 856       assert(src->is_double_xmm(), "not a double");
 857       __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
 858       break;
 859     }
 860 
 861     case T_ARRAY:   // fall through
 862     case T_OBJECT:  // fall through
 863       if (UseCompressedOops && !wide) {
 864         __ movl(as_Address(to_addr), compressed_src);
 865       } else {
 866         __ movptr(as_Address(to_addr), src->as_register());
 867       }
 868       break;
 869     case T_ADDRESS:
 870       __ movptr(as_Address(to_addr), src->as_register());
 871       break;
 872     case T_INT:
 873       __ movl(as_Address(to_addr), src->as_register());
 874       break;
 875 
 876     case T_LONG: {
 877       Register from_lo = src->as_register_lo();
 878       Register from_hi = src->as_register_hi();
 879       __ movptr(as_Address_lo(to_addr), from_lo);
 880       break;
 881     }
 882 
 883     case T_BYTE:    // fall through
 884     case T_BOOLEAN: {
 885       Register src_reg = src->as_register();
 886       Address dst_addr = as_Address(to_addr);
 887       assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
 888       __ movb(dst_addr, src_reg);
 889       break;
 890     }
 891 
 892     case T_CHAR:    // fall through
 893     case T_SHORT:
 894       __ movw(as_Address(to_addr), src->as_register());
 895       break;
 896 
 897     default:
 898       ShouldNotReachHere();
 899   }
 900   if (info != nullptr) {
 901     add_debug_info_for_null_check(null_check_here, info);
 902   }
 903 
 904   if (patch_code != lir_patch_none) {
 905     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
 906   }
 907 }
 908 
 909 
 910 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 911   assert(src->is_stack(), "should not call otherwise");
 912   assert(dest->is_register(), "should not call otherwise");
 913 
 914   if (dest->is_single_cpu()) {
 915     if (is_reference_type(type)) {
 916       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 917       __ verify_oop(dest->as_register());
 918     } else if (type == T_METADATA || type == T_ADDRESS) {
 919       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 920     } else {
 921       __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 922     }
 923 
 924   } else if (dest->is_double_cpu()) {
 925     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 926     Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
 927     __ movptr(dest->as_register_lo(), src_addr_LO);
 928 
 929   } else if (dest->is_single_xmm()) {
 930     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 931     __ movflt(dest->as_xmm_float_reg(), src_addr);
 932 
 933   } else if (dest->is_double_xmm()) {
 934     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 935     __ movdbl(dest->as_xmm_double_reg(), src_addr);
 936 
 937   } else {
 938     ShouldNotReachHere();
 939   }
 940 }
 941 
 942 
 943 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 944   if (src->is_single_stack()) {
 945     if (is_reference_type(type)) {
 946       __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
 947       __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
 948     } else {
 949       //no pushl on 64bits
 950       __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
 951       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
 952     }
 953 
 954   } else if (src->is_double_stack()) {
 955     __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
 956     __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
 957 
 958   } else {
 959     ShouldNotReachHere();
 960   }
 961 }
 962 
 963 
 964 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 965   assert(src->is_address(), "should not call otherwise");
 966   assert(dest->is_register(), "should not call otherwise");
 967 
 968   LIR_Address* addr = src->as_address_ptr();
 969   Address from_addr = as_Address(addr);
 970 
 971   if (addr->base()->type() == T_OBJECT) {
 972     __ verify_oop(addr->base()->as_pointer_register());
 973   }
 974 
 975   switch (type) {
 976     case T_BOOLEAN: // fall through
 977     case T_BYTE:    // fall through
 978     case T_CHAR:    // fall through
 979     case T_SHORT:
 980       if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
 981         // on pre P6 processors we may get partial register stalls
 982         // so blow away the value of to_rinfo before loading a
 983         // partial word into it.  Do it here so that it precedes
 984         // the potential patch point below.
 985         __ xorptr(dest->as_register(), dest->as_register());
 986       }
 987       break;
 988    default:
 989      break;
 990   }
 991 
 992   PatchingStub* patch = nullptr;
 993   if (patch_code != lir_patch_none) {
 994     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 995     assert(from_addr.disp() != 0, "must have");
 996   }
 997   if (info != nullptr) {
 998     add_debug_info_for_null_check_here(info);
 999   }
1000 
1001   switch (type) {
1002     case T_FLOAT: {
1003       if (dest->is_single_xmm()) {
1004         __ movflt(dest->as_xmm_float_reg(), from_addr);
1005       } else {
1006         ShouldNotReachHere();
1007       }
1008       break;
1009     }
1010 
1011     case T_DOUBLE: {
1012       if (dest->is_double_xmm()) {
1013         __ movdbl(dest->as_xmm_double_reg(), from_addr);
1014       } else {
1015         ShouldNotReachHere();
1016       }
1017       break;
1018     }
1019 
1020     case T_OBJECT:  // fall through
1021     case T_ARRAY:   // fall through
1022       if (UseCompressedOops && !wide) {
1023         __ movl(dest->as_register(), from_addr);
1024       } else {
1025         __ movptr(dest->as_register(), from_addr);
1026       }
1027       break;
1028 
1029     case T_ADDRESS:
1030       __ movptr(dest->as_register(), from_addr);
1031       break;
1032     case T_INT:
1033       __ movl(dest->as_register(), from_addr);
1034       break;
1035 
1036     case T_LONG: {
1037       Register to_lo = dest->as_register_lo();
1038       Register to_hi = dest->as_register_hi();
1039       __ movptr(to_lo, as_Address_lo(addr));
1040       break;
1041     }
1042 
1043     case T_BOOLEAN: // fall through
1044     case T_BYTE: {
1045       Register dest_reg = dest->as_register();
1046       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1047       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1048         __ movsbl(dest_reg, from_addr);
1049       } else {
1050         __ movb(dest_reg, from_addr);
1051         __ shll(dest_reg, 24);
1052         __ sarl(dest_reg, 24);
1053       }
1054       break;
1055     }
1056 
1057     case T_CHAR: {
1058       Register dest_reg = dest->as_register();
1059       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1060       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1061         __ movzwl(dest_reg, from_addr);
1062       } else {
1063         __ movw(dest_reg, from_addr);
1064       }
1065       break;
1066     }
1067 
1068     case T_SHORT: {
1069       Register dest_reg = dest->as_register();
1070       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1071         __ movswl(dest_reg, from_addr);
1072       } else {
1073         __ movw(dest_reg, from_addr);
1074         __ shll(dest_reg, 16);
1075         __ sarl(dest_reg, 16);
1076       }
1077       break;
1078     }
1079 
1080     default:
1081       ShouldNotReachHere();
1082   }
1083 
1084   if (patch != nullptr) {
1085     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1086   }
1087 
1088   if (is_reference_type(type)) {
1089     if (UseCompressedOops && !wide) {
1090       __ decode_heap_oop(dest->as_register());
1091     }
1092 
1093     __ verify_oop(dest->as_register());
1094   }
1095 }
1096 
1097 
1098 NEEDS_CLEANUP; // This could be static?
1099 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1100   int elem_size = type2aelembytes(type);
1101   switch (elem_size) {
1102     case 1: return Address::times_1;
1103     case 2: return Address::times_2;
1104     case 4: return Address::times_4;
1105     case 8: return Address::times_8;
1106   }
1107   ShouldNotReachHere();
1108   return Address::no_scale;
1109 }
1110 
1111 
1112 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1113   switch (op->code()) {
1114     case lir_idiv:
1115     case lir_irem:
1116       arithmetic_idiv(op->code(),
1117                       op->in_opr1(),
1118                       op->in_opr2(),
1119                       op->in_opr3(),
1120                       op->result_opr(),
1121                       op->info());
1122       break;
1123     case lir_fmad:
1124       __ fmad(op->result_opr()->as_xmm_double_reg(),
1125               op->in_opr1()->as_xmm_double_reg(),
1126               op->in_opr2()->as_xmm_double_reg(),
1127               op->in_opr3()->as_xmm_double_reg());
1128       break;
1129     case lir_fmaf:
1130       __ fmaf(op->result_opr()->as_xmm_float_reg(),
1131               op->in_opr1()->as_xmm_float_reg(),
1132               op->in_opr2()->as_xmm_float_reg(),
1133               op->in_opr3()->as_xmm_float_reg());
1134       break;
1135     default:      ShouldNotReachHere(); break;
1136   }
1137 }
1138 
1139 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1140 #ifdef ASSERT
1141   assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1142   if (op->block() != nullptr)  _branch_target_blocks.append(op->block());
1143   if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1144 #endif
1145 
1146   if (op->cond() == lir_cond_always) {
1147     if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1148     __ jmp (*(op->label()));
1149   } else {
1150     Assembler::Condition acond = Assembler::zero;
1151     if (op->code() == lir_cond_float_branch) {
1152       assert(op->ublock() != nullptr, "must have unordered successor");
1153       __ jcc(Assembler::parity, *(op->ublock()->label()));
1154       switch(op->cond()) {
1155         case lir_cond_equal:        acond = Assembler::equal;      break;
1156         case lir_cond_notEqual:     acond = Assembler::notEqual;   break;
1157         case lir_cond_less:         acond = Assembler::below;      break;
1158         case lir_cond_lessEqual:    acond = Assembler::belowEqual; break;
1159         case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1160         case lir_cond_greater:      acond = Assembler::above;      break;
1161         default:                         ShouldNotReachHere();
1162       }
1163     } else {
1164       switch (op->cond()) {
1165         case lir_cond_equal:        acond = Assembler::equal;       break;
1166         case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
1167         case lir_cond_less:         acond = Assembler::less;        break;
1168         case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
1169         case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1170         case lir_cond_greater:      acond = Assembler::greater;     break;
1171         case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
1172         case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
1173         default:                         ShouldNotReachHere();
1174       }
1175     }
1176     __ jcc(acond,*(op->label()));
1177   }
1178 }
1179 
1180 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1181   LIR_Opr src  = op->in_opr();
1182   LIR_Opr dest = op->result_opr();
1183 
1184   switch (op->bytecode()) {
1185     case Bytecodes::_i2l:
1186       __ movl2ptr(dest->as_register_lo(), src->as_register());
1187       break;
1188 
1189     case Bytecodes::_l2i:
1190       __ movl(dest->as_register(), src->as_register_lo());
1191       break;
1192 
1193     case Bytecodes::_i2b:
1194       move_regs(src->as_register(), dest->as_register());
1195       __ sign_extend_byte(dest->as_register());
1196       break;
1197 
1198     case Bytecodes::_i2c:
1199       move_regs(src->as_register(), dest->as_register());
1200       __ andl(dest->as_register(), 0xFFFF);
1201       break;
1202 
1203     case Bytecodes::_i2s:
1204       move_regs(src->as_register(), dest->as_register());
1205       __ sign_extend_short(dest->as_register());
1206       break;
1207 
1208     case Bytecodes::_f2d:
1209       __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1210       break;
1211 
1212     case Bytecodes::_d2f:
1213       __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1214       break;
1215 
1216     case Bytecodes::_i2f:
1217       __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1218       break;
1219 
1220     case Bytecodes::_i2d:
1221       __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1222       break;
1223 
1224     case Bytecodes::_l2f:
1225       __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1226       break;
1227 
1228     case Bytecodes::_l2d:
1229       __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1230       break;
1231 
1232     case Bytecodes::_f2i:
1233       __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1234       break;
1235 
1236     case Bytecodes::_d2i:
1237       __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1238       break;
1239 
1240     case Bytecodes::_f2l:
1241       __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1242       break;
1243 
1244     case Bytecodes::_d2l:
1245       __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1246       break;
1247 
1248     default: ShouldNotReachHere();
1249   }
1250 }
1251 
1252 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1253   if (op->init_check()) {
1254     add_debug_info_for_null_check_here(op->stub()->info());
1255     // init_state needs acquire, but x86 is TSO, and so we are already good.
1256     __ cmpb(Address(op->klass()->as_register(),
1257                     InstanceKlass::init_state_offset()),
1258                     InstanceKlass::fully_initialized);
1259     __ jcc(Assembler::notEqual, *op->stub()->entry());
1260   }
1261   __ allocate_object(op->obj()->as_register(),
1262                      op->tmp1()->as_register(),
1263                      op->tmp2()->as_register(),
1264                      op->header_size(),
1265                      op->object_size(),
1266                      op->klass()->as_register(),
1267                      *op->stub()->entry());
1268   __ bind(*op->stub()->continuation());
1269 }
1270 
1271 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1272   Register len =  op->len()->as_register();
1273   __ movslq(len, len);
1274 
1275   if (UseSlowPath || op->always_slow_path() ||
1276       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1277       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1278     __ jmp(*op->stub()->entry());
1279   } else {
1280     Register tmp1 = op->tmp1()->as_register();
1281     Register tmp2 = op->tmp2()->as_register();
1282     Register tmp3 = op->tmp3()->as_register();
1283     if (len == tmp1) {
1284       tmp1 = tmp3;
1285     } else if (len == tmp2) {
1286       tmp2 = tmp3;
1287     } else if (len == tmp3) {
1288       // everything is ok
1289     } else {
1290       __ mov(tmp3, len);
1291     }
1292     __ allocate_array(op->obj()->as_register(),
1293                       len,
1294                       tmp1,
1295                       tmp2,
1296                       arrayOopDesc::base_offset_in_bytes(op->type()),
1297                       array_element_size(op->type()),
1298                       op->klass()->as_register(),
1299                       *op->stub()->entry(),
1300                       op->zero_array());
1301   }
1302   __ bind(*op->stub()->continuation());
1303 }
1304 
1305 void LIR_Assembler::type_profile_helper(Register mdo,
1306                                         ciMethodData *md, ciProfileData *data,
1307                                         Register recv, Label* update_done) {
1308   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1309     Label next_test;
1310     // See if the receiver is receiver[n].
1311     __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1312     __ jccb(Assembler::notEqual, next_test);
1313     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1314     __ addptr(data_addr, DataLayout::counter_increment);
1315     __ jmp(*update_done);
1316     __ bind(next_test);
1317   }
1318 
1319   // Didn't find receiver; find next empty slot and fill it in
1320   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1321     Label next_test;
1322     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1323     __ cmpptr(recv_addr, NULL_WORD);
1324     __ jccb(Assembler::notEqual, next_test);
1325     __ movptr(recv_addr, recv);
1326     __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1327     __ jmp(*update_done);
1328     __ bind(next_test);
1329   }
1330 }
1331 
1332 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1333   // we always need a stub for the failure case.
1334   CodeStub* stub = op->stub();
1335   Register obj = op->object()->as_register();
1336   Register k_RInfo = op->tmp1()->as_register();
1337   Register klass_RInfo = op->tmp2()->as_register();
1338   Register dst = op->result_opr()->as_register();
1339   ciKlass* k = op->klass();
1340   Register Rtmp1 = noreg;
1341   Register tmp_load_klass = rscratch1;
1342 
1343   // check if it needs to be profiled
1344   ciMethodData* md = nullptr;
1345   ciProfileData* data = nullptr;
1346 
1347   if (op->should_profile()) {
1348     ciMethod* method = op->profiled_method();
1349     assert(method != nullptr, "Should have method");
1350     int bci = op->profiled_bci();
1351     md = method->method_data_or_null();
1352     assert(md != nullptr, "Sanity");
1353     data = md->bci_to_data(bci);
1354     assert(data != nullptr,                "need data for type check");
1355     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1356   }
1357   Label* success_target = success;
1358   Label* failure_target = failure;
1359 
1360   if (obj == k_RInfo) {
1361     k_RInfo = dst;
1362   } else if (obj == klass_RInfo) {
1363     klass_RInfo = dst;
1364   }
1365   if (k->is_loaded() && !UseCompressedClassPointers) {
1366     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1367   } else {
1368     Rtmp1 = op->tmp3()->as_register();
1369     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1370   }
1371 
1372   assert_different_registers(obj, k_RInfo, klass_RInfo);
1373 
1374   if (op->need_null_check()) {
1375     __ testptr(obj, obj);
1376     if (op->should_profile()) {
1377       Label not_null;
1378       Register mdo  = klass_RInfo;
1379       __ mov_metadata(mdo, md->constant_encoding());
1380       __ jccb(Assembler::notEqual, not_null);
1381       // Object is null; update MDO and exit
1382       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1383       int header_bits = BitData::null_seen_byte_constant();
1384       __ orb(data_addr, header_bits);
1385       __ jmp(*obj_is_null);
1386       __ bind(not_null);
1387 
1388       Label update_done;
1389       Register recv = k_RInfo;
1390       __ load_klass(recv, obj, tmp_load_klass);
1391       type_profile_helper(mdo, md, data, recv, &update_done);
1392 
1393       Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1394       __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1395 
1396       __ bind(update_done);
1397     } else {
1398       __ jcc(Assembler::equal, *obj_is_null);
1399     }
1400   }
1401 
1402   if (!k->is_loaded()) {
1403     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1404   } else {
1405     __ mov_metadata(k_RInfo, k->constant_encoding());
1406   }
1407   __ verify_oop(obj);
1408 
1409   if (op->fast_check()) {
1410     assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison");
1411     // get object class
1412     // not a safepoint as obj null check happens earlier
1413     if (UseCompressedClassPointers) {
1414       __ load_klass(Rtmp1, obj, tmp_load_klass);
1415       __ cmpptr(k_RInfo, Rtmp1);
1416     } else {
1417       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1418     }
1419     __ jcc(Assembler::notEqual, *failure_target);
1420     // successful cast, fall through to profile or jump
1421   } else {
1422     // get object class
1423     // not a safepoint as obj null check happens earlier
1424     __ load_klass(klass_RInfo, obj, tmp_load_klass);
1425     if (k->is_loaded()) {
1426       // See if we get an immediate positive hit
1427       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1428       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1429         __ jcc(Assembler::notEqual, *failure_target);
1430         // successful cast, fall through to profile or jump
1431       } else {
1432         // See if we get an immediate positive hit
1433         __ jcc(Assembler::equal, *success_target);
1434         // check for self
1435         if (k->is_loaded() && k->is_obj_array_klass()) {
1436           // For a direct pointer comparison, we need the refined array klass pointer
1437           ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass());
1438           __ mov_metadata(tmp_load_klass, k_refined->constant_encoding());
1439           __ cmpptr(klass_RInfo, tmp_load_klass);
1440         } else {
1441           __ cmpptr(klass_RInfo, k_RInfo);
1442         }
1443         __ jcc(Assembler::equal, *success_target);
1444 
1445         __ push_ppx(klass_RInfo);
1446         __ push_ppx(k_RInfo);
1447         __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1448         __ pop_ppx(klass_RInfo);
1449         __ pop_ppx(klass_RInfo);
1450         // result is a boolean
1451         __ testl(klass_RInfo, klass_RInfo);
1452         __ jcc(Assembler::equal, *failure_target);
1453         // successful cast, fall through to profile or jump
1454       }
1455     } else {
1456       // perform the fast part of the checking logic
1457       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1458       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1459       __ push_ppx(klass_RInfo);
1460       __ push_ppx(k_RInfo);
1461       __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1462       __ pop_ppx(klass_RInfo);
1463       __ pop_ppx(k_RInfo);
1464       // result is a boolean
1465       __ testl(k_RInfo, k_RInfo);
1466       __ jcc(Assembler::equal, *failure_target);
1467       // successful cast, fall through to profile or jump
1468     }
1469   }
1470   __ jmp(*success);
1471 }
1472 
1473 
1474 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1475   Register tmp_load_klass = rscratch1;
1476   LIR_Code code = op->code();
1477   if (code == lir_store_check) {
1478     Register value = op->object()->as_register();
1479     Register array = op->array()->as_register();
1480     Register k_RInfo = op->tmp1()->as_register();
1481     Register klass_RInfo = op->tmp2()->as_register();
1482     Register Rtmp1 = op->tmp3()->as_register();
1483 
1484     CodeStub* stub = op->stub();
1485 
1486     // check if it needs to be profiled
1487     ciMethodData* md = nullptr;
1488     ciProfileData* data = nullptr;
1489 
1490     if (op->should_profile()) {
1491       ciMethod* method = op->profiled_method();
1492       assert(method != nullptr, "Should have method");
1493       int bci = op->profiled_bci();
1494       md = method->method_data_or_null();
1495       assert(md != nullptr, "Sanity");
1496       data = md->bci_to_data(bci);
1497       assert(data != nullptr,                "need data for type check");
1498       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1499     }
1500     Label done;
1501     Label* success_target = &done;
1502     Label* failure_target = stub->entry();
1503 
1504     __ testptr(value, value);
1505     if (op->should_profile()) {
1506       Label not_null;
1507       Register mdo  = klass_RInfo;
1508       __ mov_metadata(mdo, md->constant_encoding());
1509       __ jccb(Assembler::notEqual, not_null);
1510       // Object is null; update MDO and exit
1511       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1512       int header_bits = BitData::null_seen_byte_constant();
1513       __ orb(data_addr, header_bits);
1514       __ jmp(done);
1515       __ bind(not_null);
1516 
1517       Label update_done;
1518       Register recv = k_RInfo;
1519       __ load_klass(recv, value, tmp_load_klass);
1520       type_profile_helper(mdo, md, data, recv, &update_done);
1521 
1522       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1523       __ addptr(counter_addr, DataLayout::counter_increment);
1524       __ bind(update_done);
1525     } else {
1526       __ jcc(Assembler::equal, done);
1527     }
1528 
1529     add_debug_info_for_null_check_here(op->info_for_exception());
1530     __ load_klass(k_RInfo, array, tmp_load_klass);
1531     __ load_klass(klass_RInfo, value, tmp_load_klass);
1532 
1533     // get instance klass (it's already uncompressed)
1534     __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1535     // perform the fast part of the checking logic
1536     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1537     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1538     __ push_ppx(klass_RInfo);
1539     __ push_ppx(k_RInfo);
1540     __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
1541     __ pop_ppx(klass_RInfo);
1542     __ pop_ppx(k_RInfo);
1543     // result is a boolean
1544     __ testl(k_RInfo, k_RInfo);
1545     __ jcc(Assembler::equal, *failure_target);
1546     // fall through to the success case
1547 
1548     __ bind(done);
1549   } else
1550     if (code == lir_checkcast) {
1551       Register obj = op->object()->as_register();
1552       Register dst = op->result_opr()->as_register();
1553       Label success;
1554       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1555       __ bind(success);
1556       if (dst != obj) {
1557         __ mov(dst, obj);
1558       }
1559     } else
1560       if (code == lir_instanceof) {
1561         Register obj = op->object()->as_register();
1562         Register dst = op->result_opr()->as_register();
1563         Label success, failure, done;
1564         emit_typecheck_helper(op, &success, &failure, &failure);
1565         __ bind(failure);
1566         __ xorptr(dst, dst);
1567         __ jmpb(done);
1568         __ bind(success);
1569         __ movptr(dst, 1);
1570         __ bind(done);
1571       } else {
1572         ShouldNotReachHere();
1573       }
1574 
1575 }
1576 
1577 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1578   // We are loading/storing from/to an array that *may* be a flat array (the
1579   // declared type is Object[], abstract[], interface[] or VT.ref[]).
1580   // If this array is a flat array, take the slow path.
1581   __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1582   if (!op->value()->is_illegal()) {
1583     // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1584     // The array is not a flat array, but it might be null-free. If we are storing
1585     // a null into a null-free array, take the slow path (which will throw NPE).
1586     Label skip;
1587     __ cmpptr(op->value()->as_register(), NULL_WORD);
1588     __ jcc(Assembler::notEqual, skip);
1589     __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1590     __ bind(skip);
1591   }
1592 }
1593 
1594 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1595   // We are storing into an array that *may* be null-free (the declared type is
1596   // Object[], abstract[], interface[] or VT.ref[]).
1597   Label test_mark_word;
1598   Register tmp = op->tmp()->as_register();
1599   __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1600   __ testl(tmp, markWord::unlocked_value);
1601   __ jccb(Assembler::notZero, test_mark_word);
1602   __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1603   __ bind(test_mark_word);
1604   __ testl(tmp, markWord::null_free_array_bit_in_place);
1605 }
1606 
1607 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1608   Label L_oops_equal;
1609   Label L_oops_not_equal;
1610   Label L_end;
1611 
1612   Register left  = op->left()->as_register();
1613   Register right = op->right()->as_register();
1614 
1615   __ cmpptr(left, right);
1616   __ jcc(Assembler::equal, L_oops_equal);
1617 
1618   // (1) Null check -- if one of the operands is null, the other must not be null (because
1619   //     the two references are not equal), so they are not substitutable,
1620   //     FIXME: do null check only if the operand is nullable
1621   __ testptr(left, right);
1622   __ jcc(Assembler::zero, L_oops_not_equal);
1623 
1624   ciKlass* left_klass = op->left_klass();
1625   ciKlass* right_klass = op->right_klass();
1626 
1627   // (2) Inline type check -- if either of the operands is not a inline type,
1628   //     they are not substitutable. We do this only if we are not sure that the
1629   //     operands are inline type
1630   if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1631       !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1632     Register tmp1  = op->tmp1()->as_register();
1633     __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1634     __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1635     __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1636     __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1637     __ jcc(Assembler::notEqual, L_oops_not_equal);
1638   }
1639 
1640   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1641   if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1642     // No need to load klass -- the operands are statically known to be the same inline klass.
1643     __ jmp(*op->stub()->entry());
1644   } else {
1645     Register left_klass_op = op->left_klass_op()->as_register();
1646     Register right_klass_op = op->right_klass_op()->as_register();
1647 
1648     if (UseCompressedClassPointers) {
1649       __ movl(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1650       __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1651       __ cmpl(left_klass_op, right_klass_op);
1652     } else {
1653       __ movptr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1654       __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1655       __ cmpptr(left_klass_op, right_klass_op);
1656     }
1657 
1658     __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1659     // fall through to L_oops_not_equal
1660   }
1661 
1662   __ bind(L_oops_not_equal);
1663   move(op->not_equal_result(), op->result_opr());
1664   __ jmp(L_end);
1665 
1666   __ bind(L_oops_equal);
1667   move(op->equal_result(), op->result_opr());
1668   __ jmp(L_end);
1669 
1670   // We've returned from the stub. RAX contains 0x0 IFF the two
1671   // operands are not substitutable. (Don't compare against 0x1 in case the
1672   // C compiler is naughty)
1673   __ bind(*op->stub()->continuation());
1674   __ cmpl(rax, 0);
1675   __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1676   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1677   // fall-through
1678   __ bind(L_end);
1679 }
1680 
1681 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1682   if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1683     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1684     Register newval = op->new_value()->as_register();
1685     Register cmpval = op->cmp_value()->as_register();
1686     assert(cmpval == rax, "wrong register");
1687     assert(newval != noreg, "new val must be register");
1688     assert(cmpval != newval, "cmp and new values must be in different registers");
1689     assert(cmpval != addr, "cmp and addr must be in different registers");
1690     assert(newval != addr, "new value and addr must be in different registers");
1691 
1692     if (op->code() == lir_cas_obj) {
1693       if (UseCompressedOops) {
1694         __ encode_heap_oop(cmpval);
1695         __ mov(rscratch1, newval);
1696         __ encode_heap_oop(rscratch1);
1697         __ lock();
1698         // cmpval (rax) is implicitly used by this instruction
1699         __ cmpxchgl(rscratch1, Address(addr, 0));
1700       } else {
1701         __ lock();
1702         __ cmpxchgptr(newval, Address(addr, 0));
1703       }
1704     } else {
1705       assert(op->code() == lir_cas_int, "lir_cas_int expected");
1706       __ lock();
1707       __ cmpxchgl(newval, Address(addr, 0));
1708     }
1709   } else if (op->code() == lir_cas_long) {
1710     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1711     Register newval = op->new_value()->as_register_lo();
1712     Register cmpval = op->cmp_value()->as_register_lo();
1713     assert(cmpval == rax, "wrong register");
1714     assert(newval != noreg, "new val must be register");
1715     assert(cmpval != newval, "cmp and new values must be in different registers");
1716     assert(cmpval != addr, "cmp and addr must be in different registers");
1717     assert(newval != addr, "new value and addr must be in different registers");
1718     __ lock();
1719     __ cmpxchgq(newval, Address(addr, 0));
1720   } else {
1721     Unimplemented();
1722   }
1723 }
1724 
1725 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1726   assert(dst->is_cpu_register(), "must be");
1727   assert(dst->type() == src->type(), "must be");
1728 
1729   if (src->is_cpu_register()) {
1730     reg2reg(src, dst);
1731   } else if (src->is_stack()) {
1732     stack2reg(src, dst, dst->type());
1733   } else if (src->is_constant()) {
1734     const2reg(src, dst, lir_patch_none, nullptr);
1735   } else {
1736     ShouldNotReachHere();
1737   }
1738 }
1739 
1740 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1741                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1742   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1743 
1744   Assembler::Condition acond, ncond;
1745   switch (condition) {
1746     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1747     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1748     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1749     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1750     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1751     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1752     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1753     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1754     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1755                                 ShouldNotReachHere();
1756   }
1757 
1758   if (opr1->is_cpu_register()) {
1759     reg2reg(opr1, result);
1760   } else if (opr1->is_stack()) {
1761     stack2reg(opr1, result, result->type());
1762   } else if (opr1->is_constant()) {
1763     const2reg(opr1, result, lir_patch_none, nullptr);
1764   } else {
1765     ShouldNotReachHere();
1766   }
1767 
1768   if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1769     // optimized version that does not require a branch
1770     if (opr2->is_single_cpu()) {
1771       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1772       __ cmov(ncond, result->as_register(), opr2->as_register());
1773     } else if (opr2->is_double_cpu()) {
1774       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1775       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1776       __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1777     } else if (opr2->is_single_stack()) {
1778       __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1779     } else if (opr2->is_double_stack()) {
1780       __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1781     } else {
1782       ShouldNotReachHere();
1783     }
1784 
1785   } else {
1786     Label skip;
1787     __ jccb(acond, skip);
1788     if (opr2->is_cpu_register()) {
1789       reg2reg(opr2, result);
1790     } else if (opr2->is_stack()) {
1791       stack2reg(opr2, result, result->type());
1792     } else if (opr2->is_constant()) {
1793       const2reg(opr2, result, lir_patch_none, nullptr);
1794     } else {
1795       ShouldNotReachHere();
1796     }
1797     __ bind(skip);
1798   }
1799 }
1800 
1801 
1802 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1803   assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1804 
1805   if (left->is_single_cpu()) {
1806     assert(left == dest, "left and dest must be equal");
1807     Register lreg = left->as_register();
1808 
1809     if (right->is_single_cpu()) {
1810       // cpu register - cpu register
1811       Register rreg = right->as_register();
1812       switch (code) {
1813         case lir_add: __ addl (lreg, rreg); break;
1814         case lir_sub: __ subl (lreg, rreg); break;
1815         case lir_mul: __ imull(lreg, rreg); break;
1816         default:      ShouldNotReachHere();
1817       }
1818 
1819     } else if (right->is_stack()) {
1820       // cpu register - stack
1821       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1822       switch (code) {
1823         case lir_add: __ addl(lreg, raddr); break;
1824         case lir_sub: __ subl(lreg, raddr); break;
1825         default:      ShouldNotReachHere();
1826       }
1827 
1828     } else if (right->is_constant()) {
1829       // cpu register - constant
1830       jint c = right->as_constant_ptr()->as_jint();
1831       switch (code) {
1832         case lir_add: {
1833           __ incrementl(lreg, c);
1834           break;
1835         }
1836         case lir_sub: {
1837           __ decrementl(lreg, c);
1838           break;
1839         }
1840         default: ShouldNotReachHere();
1841       }
1842 
1843     } else {
1844       ShouldNotReachHere();
1845     }
1846 
1847   } else if (left->is_double_cpu()) {
1848     assert(left == dest, "left and dest must be equal");
1849     Register lreg_lo = left->as_register_lo();
1850     Register lreg_hi = left->as_register_hi();
1851 
1852     if (right->is_double_cpu()) {
1853       // cpu register - cpu register
1854       Register rreg_lo = right->as_register_lo();
1855       Register rreg_hi = right->as_register_hi();
1856       assert_different_registers(lreg_lo, rreg_lo);
1857       switch (code) {
1858         case lir_add:
1859           __ addptr(lreg_lo, rreg_lo);
1860           break;
1861         case lir_sub:
1862           __ subptr(lreg_lo, rreg_lo);
1863           break;
1864         case lir_mul:
1865           __ imulq(lreg_lo, rreg_lo);
1866           break;
1867         default:
1868           ShouldNotReachHere();
1869       }
1870 
1871     } else if (right->is_constant()) {
1872       // cpu register - constant
1873       jlong c = right->as_constant_ptr()->as_jlong_bits();
1874       __ movptr(r10, (intptr_t) c);
1875       switch (code) {
1876         case lir_add:
1877           __ addptr(lreg_lo, r10);
1878           break;
1879         case lir_sub:
1880           __ subptr(lreg_lo, r10);
1881           break;
1882         default:
1883           ShouldNotReachHere();
1884       }
1885 
1886     } else {
1887       ShouldNotReachHere();
1888     }
1889 
1890   } else if (left->is_single_xmm()) {
1891     assert(left == dest, "left and dest must be equal");
1892     XMMRegister lreg = left->as_xmm_float_reg();
1893 
1894     if (right->is_single_xmm()) {
1895       XMMRegister rreg = right->as_xmm_float_reg();
1896       switch (code) {
1897         case lir_add: __ addss(lreg, rreg);  break;
1898         case lir_sub: __ subss(lreg, rreg);  break;
1899         case lir_mul: __ mulss(lreg, rreg);  break;
1900         case lir_div: __ divss(lreg, rreg);  break;
1901         default: ShouldNotReachHere();
1902       }
1903     } else {
1904       Address raddr;
1905       if (right->is_single_stack()) {
1906         raddr = frame_map()->address_for_slot(right->single_stack_ix());
1907       } else if (right->is_constant()) {
1908         // hack for now
1909         raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1910       } else {
1911         ShouldNotReachHere();
1912       }
1913       switch (code) {
1914         case lir_add: __ addss(lreg, raddr);  break;
1915         case lir_sub: __ subss(lreg, raddr);  break;
1916         case lir_mul: __ mulss(lreg, raddr);  break;
1917         case lir_div: __ divss(lreg, raddr);  break;
1918         default: ShouldNotReachHere();
1919       }
1920     }
1921 
1922   } else if (left->is_double_xmm()) {
1923     assert(left == dest, "left and dest must be equal");
1924 
1925     XMMRegister lreg = left->as_xmm_double_reg();
1926     if (right->is_double_xmm()) {
1927       XMMRegister rreg = right->as_xmm_double_reg();
1928       switch (code) {
1929         case lir_add: __ addsd(lreg, rreg);  break;
1930         case lir_sub: __ subsd(lreg, rreg);  break;
1931         case lir_mul: __ mulsd(lreg, rreg);  break;
1932         case lir_div: __ divsd(lreg, rreg);  break;
1933         default: ShouldNotReachHere();
1934       }
1935     } else {
1936       Address raddr;
1937       if (right->is_double_stack()) {
1938         raddr = frame_map()->address_for_slot(right->double_stack_ix());
1939       } else if (right->is_constant()) {
1940         // hack for now
1941         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1942       } else {
1943         ShouldNotReachHere();
1944       }
1945       switch (code) {
1946         case lir_add: __ addsd(lreg, raddr);  break;
1947         case lir_sub: __ subsd(lreg, raddr);  break;
1948         case lir_mul: __ mulsd(lreg, raddr);  break;
1949         case lir_div: __ divsd(lreg, raddr);  break;
1950         default: ShouldNotReachHere();
1951       }
1952     }
1953 
1954   } else if (left->is_single_stack() || left->is_address()) {
1955     assert(left == dest, "left and dest must be equal");
1956 
1957     Address laddr;
1958     if (left->is_single_stack()) {
1959       laddr = frame_map()->address_for_slot(left->single_stack_ix());
1960     } else if (left->is_address()) {
1961       laddr = as_Address(left->as_address_ptr());
1962     } else {
1963       ShouldNotReachHere();
1964     }
1965 
1966     if (right->is_single_cpu()) {
1967       Register rreg = right->as_register();
1968       switch (code) {
1969         case lir_add: __ addl(laddr, rreg); break;
1970         case lir_sub: __ subl(laddr, rreg); break;
1971         default:      ShouldNotReachHere();
1972       }
1973     } else if (right->is_constant()) {
1974       jint c = right->as_constant_ptr()->as_jint();
1975       switch (code) {
1976         case lir_add: {
1977           __ incrementl(laddr, c);
1978           break;
1979         }
1980         case lir_sub: {
1981           __ decrementl(laddr, c);
1982           break;
1983         }
1984         default: ShouldNotReachHere();
1985       }
1986     } else {
1987       ShouldNotReachHere();
1988     }
1989 
1990   } else {
1991     ShouldNotReachHere();
1992   }
1993 }
1994 
1995 
1996 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1997   if (value->is_double_xmm()) {
1998     switch(code) {
1999       case lir_abs :
2000         {
2001           if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2002             __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2003           }
2004           assert(!tmp->is_valid(), "do not need temporary");
2005           __ andpd(dest->as_xmm_double_reg(),
2006                    ExternalAddress((address)double_signmask_pool),
2007                    rscratch1);
2008         }
2009         break;
2010 
2011       case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2012       // all other intrinsics are not available in the SSE instruction set, so FPU is used
2013       default      : ShouldNotReachHere();
2014     }
2015 
2016   } else if (code == lir_f2hf) {
2017     __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
2018   } else if (code == lir_hf2f) {
2019     __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
2020   } else {
2021     Unimplemented();
2022   }
2023 }
2024 
2025 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2026   // assert(left->destroys_register(), "check");
2027   if (left->is_single_cpu()) {
2028     Register reg = left->as_register();
2029     if (right->is_constant()) {
2030       int val = right->as_constant_ptr()->as_jint();
2031       switch (code) {
2032         case lir_logic_and: __ andl (reg, val); break;
2033         case lir_logic_or:  __ orl  (reg, val); break;
2034         case lir_logic_xor: __ xorl (reg, val); break;
2035         default: ShouldNotReachHere();
2036       }
2037     } else if (right->is_stack()) {
2038       // added support for stack operands
2039       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2040       switch (code) {
2041         case lir_logic_and: __ andl (reg, raddr); break;
2042         case lir_logic_or:  __ orl  (reg, raddr); break;
2043         case lir_logic_xor: __ xorl (reg, raddr); break;
2044         default: ShouldNotReachHere();
2045       }
2046     } else {
2047       Register rright = right->as_register();
2048       switch (code) {
2049         case lir_logic_and: __ andptr (reg, rright); break;
2050         case lir_logic_or : __ orptr  (reg, rright); break;
2051         case lir_logic_xor: __ xorptr (reg, rright); break;
2052         default: ShouldNotReachHere();
2053       }
2054     }
2055     move_regs(reg, dst->as_register());
2056   } else {
2057     Register l_lo = left->as_register_lo();
2058     Register l_hi = left->as_register_hi();
2059     if (right->is_constant()) {
2060       __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2061       switch (code) {
2062         case lir_logic_and:
2063           __ andq(l_lo, rscratch1);
2064           break;
2065         case lir_logic_or:
2066           __ orq(l_lo, rscratch1);
2067           break;
2068         case lir_logic_xor:
2069           __ xorq(l_lo, rscratch1);
2070           break;
2071         default: ShouldNotReachHere();
2072       }
2073     } else {
2074       Register r_lo;
2075       if (is_reference_type(right->type())) {
2076         r_lo = right->as_register();
2077       } else {
2078         r_lo = right->as_register_lo();
2079       }
2080       switch (code) {
2081         case lir_logic_and:
2082           __ andptr(l_lo, r_lo);
2083           break;
2084         case lir_logic_or:
2085           __ orptr(l_lo, r_lo);
2086           break;
2087         case lir_logic_xor:
2088           __ xorptr(l_lo, r_lo);
2089           break;
2090         default: ShouldNotReachHere();
2091       }
2092     }
2093 
2094     Register dst_lo = dst->as_register_lo();
2095     Register dst_hi = dst->as_register_hi();
2096 
2097     move_regs(l_lo, dst_lo);
2098   }
2099 }
2100 
2101 
2102 // we assume that rax, and rdx can be overwritten
2103 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2104 
2105   assert(left->is_single_cpu(),   "left must be register");
2106   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2107   assert(result->is_single_cpu(), "result must be register");
2108 
2109   //  assert(left->destroys_register(), "check");
2110   //  assert(right->destroys_register(), "check");
2111 
2112   Register lreg = left->as_register();
2113   Register dreg = result->as_register();
2114 
2115   if (right->is_constant()) {
2116     jint divisor = right->as_constant_ptr()->as_jint();
2117     assert(divisor > 0 && is_power_of_2(divisor), "must be");
2118     if (code == lir_idiv) {
2119       assert(lreg == rax, "must be rax,");
2120       assert(temp->as_register() == rdx, "tmp register must be rdx");
2121       __ cdql(); // sign extend into rdx:rax
2122       if (divisor == 2) {
2123         __ subl(lreg, rdx);
2124       } else {
2125         __ andl(rdx, divisor - 1);
2126         __ addl(lreg, rdx);
2127       }
2128       __ sarl(lreg, log2i_exact(divisor));
2129       move_regs(lreg, dreg);
2130     } else if (code == lir_irem) {
2131       Label done;
2132       __ mov(dreg, lreg);
2133       __ andl(dreg, 0x80000000 | (divisor - 1));
2134       __ jcc(Assembler::positive, done);
2135       __ decrement(dreg);
2136       __ orl(dreg, ~(divisor - 1));
2137       __ increment(dreg);
2138       __ bind(done);
2139     } else {
2140       ShouldNotReachHere();
2141     }
2142   } else {
2143     Register rreg = right->as_register();
2144     assert(lreg == rax, "left register must be rax,");
2145     assert(rreg != rdx, "right register must not be rdx");
2146     assert(temp->as_register() == rdx, "tmp register must be rdx");
2147 
2148     move_regs(lreg, rax);
2149 
2150     int idivl_offset = __ corrected_idivl(rreg);
2151     if (ImplicitDiv0Checks) {
2152       add_debug_info_for_div0(idivl_offset, info);
2153     }
2154     if (code == lir_irem) {
2155       move_regs(rdx, dreg); // result is in rdx
2156     } else {
2157       move_regs(rax, dreg);
2158     }
2159   }
2160 }
2161 
2162 
2163 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2164   if (opr1->is_single_cpu()) {
2165     Register reg1 = opr1->as_register();
2166     if (opr2->is_single_cpu()) {
2167       // cpu register - cpu register
2168       if (is_reference_type(opr1->type())) {
2169         __ cmpoop(reg1, opr2->as_register());
2170       } else {
2171         assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2172         __ cmpl(reg1, opr2->as_register());
2173       }
2174     } else if (opr2->is_stack()) {
2175       // cpu register - stack
2176       if (is_reference_type(opr1->type())) {
2177         __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2178       } else {
2179         __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2180       }
2181     } else if (opr2->is_constant()) {
2182       // cpu register - constant
2183       LIR_Const* c = opr2->as_constant_ptr();
2184       if (c->type() == T_INT) {
2185         jint i = c->as_jint();
2186         if (i == 0) {
2187           __ testl(reg1, reg1);
2188         } else {
2189           __ cmpl(reg1, i);
2190         }
2191       } else if (c->type() == T_METADATA) {
2192         // All we need for now is a comparison with null for equality.
2193         assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2194         Metadata* m = c->as_metadata();
2195         if (m == nullptr) {
2196           __ testptr(reg1, reg1);
2197         } else {
2198           ShouldNotReachHere();
2199         }
2200       } else if (is_reference_type(c->type())) {
2201         // In 64bit oops are single register
2202         jobject o = c->as_jobject();
2203         if (o == nullptr) {
2204           __ testptr(reg1, reg1);
2205         } else {
2206           __ cmpoop(reg1, o, rscratch1);
2207         }
2208       } else {
2209         fatal("unexpected type: %s", basictype_to_str(c->type()));
2210       }
2211       // cpu register - address
2212     } else if (opr2->is_address()) {
2213       if (op->info() != nullptr) {
2214         add_debug_info_for_null_check_here(op->info());
2215       }
2216       __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2217     } else {
2218       ShouldNotReachHere();
2219     }
2220 
2221   } else if(opr1->is_double_cpu()) {
2222     Register xlo = opr1->as_register_lo();
2223     Register xhi = opr1->as_register_hi();
2224     if (opr2->is_double_cpu()) {
2225       __ cmpptr(xlo, opr2->as_register_lo());
2226     } else if (opr2->is_constant()) {
2227       // cpu register - constant 0
2228       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2229       __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2230     } else {
2231       ShouldNotReachHere();
2232     }
2233 
2234   } else if (opr1->is_single_xmm()) {
2235     XMMRegister reg1 = opr1->as_xmm_float_reg();
2236     if (opr2->is_single_xmm()) {
2237       // xmm register - xmm register
2238       __ ucomiss(reg1, opr2->as_xmm_float_reg());
2239     } else if (opr2->is_stack()) {
2240       // xmm register - stack
2241       __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2242     } else if (opr2->is_constant()) {
2243       // xmm register - constant
2244       __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2245     } else if (opr2->is_address()) {
2246       // xmm register - address
2247       if (op->info() != nullptr) {
2248         add_debug_info_for_null_check_here(op->info());
2249       }
2250       __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2251     } else {
2252       ShouldNotReachHere();
2253     }
2254 
2255   } else if (opr1->is_double_xmm()) {
2256     XMMRegister reg1 = opr1->as_xmm_double_reg();
2257     if (opr2->is_double_xmm()) {
2258       // xmm register - xmm register
2259       __ ucomisd(reg1, opr2->as_xmm_double_reg());
2260     } else if (opr2->is_stack()) {
2261       // xmm register - stack
2262       __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2263     } else if (opr2->is_constant()) {
2264       // xmm register - constant
2265       __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2266     } else if (opr2->is_address()) {
2267       // xmm register - address
2268       if (op->info() != nullptr) {
2269         add_debug_info_for_null_check_here(op->info());
2270       }
2271       __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2272     } else {
2273       ShouldNotReachHere();
2274     }
2275 
2276   } else if (opr1->is_address() && opr2->is_constant()) {
2277     LIR_Const* c = opr2->as_constant_ptr();
2278     if (is_reference_type(c->type())) {
2279       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2280       __ movoop(rscratch1, c->as_jobject());
2281     }
2282     if (op->info() != nullptr) {
2283       add_debug_info_for_null_check_here(op->info());
2284     }
2285     // special case: address - constant
2286     LIR_Address* addr = opr1->as_address_ptr();
2287     if (c->type() == T_INT) {
2288       __ cmpl(as_Address(addr), c->as_jint());
2289     } else if (is_reference_type(c->type())) {
2290       // %%% Make this explode if addr isn't reachable until we figure out a
2291       // better strategy by giving noreg as the temp for as_Address
2292       __ cmpoop(rscratch1, as_Address(addr, noreg));
2293     } else {
2294       ShouldNotReachHere();
2295     }
2296 
2297   } else {
2298     ShouldNotReachHere();
2299   }
2300 }
2301 
2302 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2303   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2304     if (left->is_single_xmm()) {
2305       assert(right->is_single_xmm(), "must match");
2306       __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2307     } else if (left->is_double_xmm()) {
2308       assert(right->is_double_xmm(), "must match");
2309       __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2310 
2311     } else {
2312       ShouldNotReachHere();
2313     }
2314   } else {
2315     assert(code == lir_cmp_l2i, "check");
2316     Label done;
2317     Register dest = dst->as_register();
2318     __ cmpptr(left->as_register_lo(), right->as_register_lo());
2319     __ movl(dest, -1);
2320     __ jccb(Assembler::less, done);
2321     __ setb(Assembler::notZero, dest);
2322     __ movzbl(dest, dest);
2323     __ bind(done);
2324   }
2325 }
2326 
2327 
2328 void LIR_Assembler::align_call(LIR_Code code) {
2329   // make sure that the displacement word of the call ends up word aligned
2330   int offset = __ offset();
2331   switch (code) {
2332   case lir_static_call:
2333   case lir_optvirtual_call:
2334   case lir_dynamic_call:
2335     offset += NativeCall::displacement_offset;
2336     break;
2337   case lir_icvirtual_call:
2338     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2339     break;
2340   default: ShouldNotReachHere();
2341   }
2342   __ align(BytesPerWord, offset);
2343 }
2344 
2345 
2346 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2347   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2348          "must be aligned");
2349   __ call(AddressLiteral(op->addr(), rtype));
2350   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2351   __ post_call_nop();
2352 }
2353 
2354 
2355 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2356   __ ic_call(op->addr());
2357   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2358   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2359          "must be aligned");
2360   __ post_call_nop();
2361 }
2362 
2363 
2364 void LIR_Assembler::emit_static_call_stub() {
2365   address call_pc = __ pc();
2366   address stub = __ start_a_stub(call_stub_size());
2367   if (stub == nullptr) {
2368     bailout("static call stub overflow");
2369     return;
2370   }
2371 
2372   int start = __ offset();
2373 
2374   // make sure that the displacement word of the call ends up word aligned
2375   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2376   __ relocate(static_stub_Relocation::spec(call_pc));
2377   __ mov_metadata(rbx, (Metadata*)nullptr);
2378   // must be set to -1 at code generation time
2379   assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2380   // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2381   __ jump(RuntimeAddress(__ pc()));
2382 
2383   assert(__ offset() - start <= call_stub_size(), "stub too big");
2384   __ end_a_stub();
2385 }
2386 
2387 
2388 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2389   assert(exceptionOop->as_register() == rax, "must match");
2390   assert(exceptionPC->as_register() == rdx, "must match");
2391 
2392   // exception object is not added to oop map by LinearScan
2393   // (LinearScan assumes that no oops are in fixed registers)
2394   info->add_register_oop(exceptionOop);
2395   StubId unwind_id;
2396 
2397   // get current pc information
2398   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2399   int pc_for_athrow_offset = __ offset();
2400   InternalAddress pc_for_athrow(__ pc());
2401   __ lea(exceptionPC->as_register(), pc_for_athrow);
2402   add_call_info(pc_for_athrow_offset, info); // for exception handler
2403 
2404   __ verify_not_null_oop(rax);
2405   // search an exception handler (rax: exception oop, rdx: throwing pc)
2406   if (compilation()->has_fpu_code()) {
2407     unwind_id = StubId::c1_handle_exception_id;
2408   } else {
2409     unwind_id = StubId::c1_handle_exception_nofpu_id;
2410   }
2411   __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2412 
2413   // enough room for two byte trap
2414   __ nop();
2415 }
2416 
2417 
2418 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2419   assert(exceptionOop->as_register() == rax, "must match");
2420 
2421   __ jmp(_unwind_handler_entry);
2422 }
2423 
2424 
2425 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2426 
2427   // optimized version for linear scan:
2428   // * count must be already in ECX (guaranteed by LinearScan)
2429   // * left and dest must be equal
2430   // * tmp must be unused
2431   assert(count->as_register() == SHIFT_count, "count must be in ECX");
2432   assert(left == dest, "left and dest must be equal");
2433   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2434 
2435   if (left->is_single_cpu()) {
2436     Register value = left->as_register();
2437     assert(value != SHIFT_count, "left cannot be ECX");
2438 
2439     switch (code) {
2440       case lir_shl:  __ shll(value); break;
2441       case lir_shr:  __ sarl(value); break;
2442       case lir_ushr: __ shrl(value); break;
2443       default: ShouldNotReachHere();
2444     }
2445   } else if (left->is_double_cpu()) {
2446     Register lo = left->as_register_lo();
2447     Register hi = left->as_register_hi();
2448     assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2449     switch (code) {
2450       case lir_shl:  __ shlptr(lo);        break;
2451       case lir_shr:  __ sarptr(lo);        break;
2452       case lir_ushr: __ shrptr(lo);        break;
2453       default: ShouldNotReachHere();
2454     }
2455   } else {
2456     ShouldNotReachHere();
2457   }
2458 }
2459 
2460 
2461 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2462   if (dest->is_single_cpu()) {
2463     // first move left into dest so that left is not destroyed by the shift
2464     Register value = dest->as_register();
2465     count = count & 0x1F; // Java spec
2466 
2467     move_regs(left->as_register(), value);
2468     switch (code) {
2469       case lir_shl:  __ shll(value, count); break;
2470       case lir_shr:  __ sarl(value, count); break;
2471       case lir_ushr: __ shrl(value, count); break;
2472       default: ShouldNotReachHere();
2473     }
2474   } else if (dest->is_double_cpu()) {
2475     // first move left into dest so that left is not destroyed by the shift
2476     Register value = dest->as_register_lo();
2477     count = count & 0x1F; // Java spec
2478 
2479     move_regs(left->as_register_lo(), value);
2480     switch (code) {
2481       case lir_shl:  __ shlptr(value, count); break;
2482       case lir_shr:  __ sarptr(value, count); break;
2483       case lir_ushr: __ shrptr(value, count); break;
2484       default: ShouldNotReachHere();
2485     }
2486   } else {
2487     ShouldNotReachHere();
2488   }
2489 }
2490 
2491 
2492 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2493   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2494   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2495   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2496   __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2497 }
2498 
2499 
2500 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2501   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2502   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2503   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2504   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2505 }
2506 
2507 
2508 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2509   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2510   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2511   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2512   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2513 }
2514 
2515 
2516 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2517   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2518   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2519   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2520   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2521 }
2522 
2523 
2524 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2525   if (null_check) {
2526     __ testptr(obj, obj);
2527     __ jcc(Assembler::zero, *slow_path->entry());
2528   }
2529   if (is_dest) {
2530     __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2531     // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2532     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2533   } else {
2534     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2535   }
2536 }
2537 
2538 
2539 // This code replaces a call to arraycopy; no exception may
2540 // be thrown in this code, they must be thrown in the System.arraycopy
2541 // activation frame; we could save some checks if this would not be the case
2542 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2543   ciArrayKlass* default_type = op->expected_type();
2544   Register src = op->src()->as_register();
2545   Register dst = op->dst()->as_register();
2546   Register src_pos = op->src_pos()->as_register();
2547   Register dst_pos = op->dst_pos()->as_register();
2548   Register length  = op->length()->as_register();
2549   Register tmp = op->tmp()->as_register();
2550   Register tmp_load_klass = rscratch1;
2551   Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2552 
2553   CodeStub* stub = op->stub();
2554   int flags = op->flags();
2555   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2556   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2557 
2558   if (flags & LIR_OpArrayCopy::always_slow_path) {
2559     __ jmp(*stub->entry());
2560     __ bind(*stub->continuation());
2561     return;
2562   }
2563 
2564   // if we don't know anything, just go through the generic arraycopy
2565   if (default_type == nullptr) {
2566     // save outgoing arguments on stack in case call to System.arraycopy is needed
2567     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2568     // for interpreter calling conventions. Now we have to do it in new style conventions.
2569     // For the moment until C1 gets the new register allocator I just force all the
2570     // args to the right place (except the register args) and then on the back side
2571     // reload the register args properly if we go slow path. Yuck
2572 
2573     // These are proper for the calling convention
2574     store_parameter(length, 2);
2575     store_parameter(dst_pos, 1);
2576     store_parameter(dst, 0);
2577 
2578     // these are just temporary placements until we need to reload
2579     store_parameter(src_pos, 3);
2580     store_parameter(src, 4);
2581 
2582     address copyfunc_addr = StubRoutines::generic_arraycopy();
2583     assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2584 
2585     // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2586     // The arguments are in java calling convention so we can trivially shift them to C
2587     // convention
2588     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2589     __ mov(c_rarg0, j_rarg0);
2590     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2591     __ mov(c_rarg1, j_rarg1);
2592     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2593     __ mov(c_rarg2, j_rarg2);
2594     assert_different_registers(c_rarg3, j_rarg4);
2595     __ mov(c_rarg3, j_rarg3);
2596 #ifdef _WIN64
2597     // Allocate abi space for args but be sure to keep stack aligned
2598     __ subptr(rsp, 6*wordSize);
2599     store_parameter(j_rarg4, 4);
2600 #ifndef PRODUCT
2601     if (PrintC1Statistics) {
2602       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2603     }
2604 #endif
2605     __ call(RuntimeAddress(copyfunc_addr));
2606     __ addptr(rsp, 6*wordSize);
2607 #else
2608     __ mov(c_rarg4, j_rarg4);
2609 #ifndef PRODUCT
2610     if (PrintC1Statistics) {
2611       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2612     }
2613 #endif
2614     __ call(RuntimeAddress(copyfunc_addr));
2615 #endif // _WIN64
2616 
2617     __ testl(rax, rax);
2618     __ jcc(Assembler::equal, *stub->continuation());
2619 
2620     __ mov(tmp, rax);
2621     __ xorl(tmp, -1);
2622 
2623     // Reload values from the stack so they are where the stub
2624     // expects them.
2625     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2626     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2627     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2628     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2629     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2630 
2631     __ subl(length, tmp);
2632     __ addl(src_pos, tmp);
2633     __ addl(dst_pos, tmp);
2634     __ jmp(*stub->entry());
2635 
2636     __ bind(*stub->continuation());
2637     return;
2638   }
2639 
2640   // Handle inline type arrays
2641   if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2642     arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2643   }
2644   if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2645     arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2646   }
2647 
2648   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2649 
2650   int elem_size = type2aelembytes(basic_type);
2651   Address::ScaleFactor scale;
2652 
2653   switch (elem_size) {
2654     case 1 :
2655       scale = Address::times_1;
2656       break;
2657     case 2 :
2658       scale = Address::times_2;
2659       break;
2660     case 4 :
2661       scale = Address::times_4;
2662       break;
2663     case 8 :
2664       scale = Address::times_8;
2665       break;
2666     default:
2667       scale = Address::no_scale;
2668       ShouldNotReachHere();
2669   }
2670 
2671   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2672   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2673 
2674   // length and pos's are all sign extended at this point on 64bit
2675 
2676   // test for null
2677   if (flags & LIR_OpArrayCopy::src_null_check) {
2678     __ testptr(src, src);
2679     __ jcc(Assembler::zero, *stub->entry());
2680   }
2681   if (flags & LIR_OpArrayCopy::dst_null_check) {
2682     __ testptr(dst, dst);
2683     __ jcc(Assembler::zero, *stub->entry());
2684   }
2685 
2686   // If the compiler was not able to prove that exact type of the source or the destination
2687   // of the arraycopy is an array type, check at runtime if the source or the destination is
2688   // an instance type.
2689   if (flags & LIR_OpArrayCopy::type_check) {
2690     if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2691       __ load_klass(tmp, dst, tmp_load_klass);
2692       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2693       __ jcc(Assembler::greaterEqual, *stub->entry());
2694     }
2695 
2696     if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2697       __ load_klass(tmp, src, tmp_load_klass);
2698       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2699       __ jcc(Assembler::greaterEqual, *stub->entry());
2700     }
2701   }
2702 
2703   // check if negative
2704   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2705     __ testl(src_pos, src_pos);
2706     __ jcc(Assembler::less, *stub->entry());
2707   }
2708   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2709     __ testl(dst_pos, dst_pos);
2710     __ jcc(Assembler::less, *stub->entry());
2711   }
2712 
2713   if (flags & LIR_OpArrayCopy::src_range_check) {
2714     __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2715     __ cmpl(tmp, src_length_addr);
2716     __ jcc(Assembler::above, *stub->entry());
2717   }
2718   if (flags & LIR_OpArrayCopy::dst_range_check) {
2719     __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2720     __ cmpl(tmp, dst_length_addr);
2721     __ jcc(Assembler::above, *stub->entry());
2722   }
2723 
2724   if (flags & LIR_OpArrayCopy::length_positive_check) {
2725     __ testl(length, length);
2726     __ jcc(Assembler::less, *stub->entry());
2727   }
2728 
2729   __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2730   __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2731 
2732   if (flags & LIR_OpArrayCopy::type_check) {
2733     // We don't know the array types are compatible
2734     if (basic_type != T_OBJECT) {
2735       // Simple test for basic type arrays
2736       __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2737       __ jcc(Assembler::notEqual, *stub->entry());
2738     } else {
2739       // For object arrays, if src is a sub class of dst then we can
2740       // safely do the copy.
2741       Label cont, slow;
2742 
2743       __ push_ppx(src);
2744       __ push_ppx(dst);
2745 
2746       __ load_klass(src, src, tmp_load_klass);
2747       __ load_klass(dst, dst, tmp_load_klass);
2748 
2749       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2750 
2751       __ push_ppx(src);
2752       __ push_ppx(dst);
2753       __ call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id)));
2754       __ pop_ppx(dst);
2755       __ pop_ppx(src);
2756 
2757       __ testl(src, src);
2758       __ jcc(Assembler::notEqual, cont);
2759 
2760       __ bind(slow);
2761       __ pop_ppx(dst);
2762       __ pop_ppx(src);
2763 
2764       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2765       if (copyfunc_addr != nullptr) { // use stub if available
2766         // src is not a sub class of dst so we have to do a
2767         // per-element check.
2768 
2769         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2770         if ((flags & mask) != mask) {
2771           // Check that at least both of them object arrays.
2772           assert(flags & mask, "one of the two should be known to be an object array");
2773 
2774           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2775             __ load_klass(tmp, src, tmp_load_klass);
2776           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2777             __ load_klass(tmp, dst, tmp_load_klass);
2778           }
2779           int lh_offset = in_bytes(Klass::layout_helper_offset());
2780           Address klass_lh_addr(tmp, lh_offset);
2781           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2782           __ cmpl(klass_lh_addr, objArray_lh);
2783           __ jcc(Assembler::notEqual, *stub->entry());
2784         }
2785 
2786        // Spill because stubs can use any register they like and it's
2787        // easier to restore just those that we care about.
2788        store_parameter(dst, 0);
2789        store_parameter(dst_pos, 1);
2790        store_parameter(length, 2);
2791        store_parameter(src_pos, 3);
2792        store_parameter(src, 4);
2793 
2794         __ movl2ptr(length, length); //higher 32bits must be null
2795 
2796         __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2797         assert_different_registers(c_rarg0, dst, dst_pos, length);
2798         __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2799         assert_different_registers(c_rarg1, dst, length);
2800 
2801         __ mov(c_rarg2, length);
2802         assert_different_registers(c_rarg2, dst);
2803 
2804 #ifdef _WIN64
2805         // Allocate abi space for args but be sure to keep stack aligned
2806         __ subptr(rsp, 6*wordSize);
2807         __ load_klass(c_rarg3, dst, tmp_load_klass);
2808         __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2809         store_parameter(c_rarg3, 4);
2810         __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2811         __ call(RuntimeAddress(copyfunc_addr));
2812         __ addptr(rsp, 6*wordSize);
2813 #else
2814         __ load_klass(c_rarg4, dst, tmp_load_klass);
2815         __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2816         __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2817         __ call(RuntimeAddress(copyfunc_addr));
2818 #endif
2819 
2820 #ifndef PRODUCT
2821         if (PrintC1Statistics) {
2822           Label failed;
2823           __ testl(rax, rax);
2824           __ jcc(Assembler::notZero, failed);
2825           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2826           __ bind(failed);
2827         }
2828 #endif
2829 
2830         __ testl(rax, rax);
2831         __ jcc(Assembler::zero, *stub->continuation());
2832 
2833 #ifndef PRODUCT
2834         if (PrintC1Statistics) {
2835           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2836         }
2837 #endif
2838 
2839         __ mov(tmp, rax);
2840 
2841         __ xorl(tmp, -1);
2842 
2843         // Restore previously spilled arguments
2844         __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2845         __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2846         __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2847         __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2848         __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2849 
2850 
2851         __ subl(length, tmp);
2852         __ addl(src_pos, tmp);
2853         __ addl(dst_pos, tmp);
2854       }
2855 
2856       __ jmp(*stub->entry());
2857 
2858       __ bind(cont);
2859       __ pop(dst);
2860       __ pop(src);
2861     }
2862   }
2863 
2864 #ifdef ASSERT
2865   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2866     // Sanity check the known type with the incoming class.  For the
2867     // primitive case the types must match exactly with src.klass and
2868     // dst.klass each exactly matching the default type.  For the
2869     // object array case, if no type check is needed then either the
2870     // dst type is exactly the expected type and the src type is a
2871     // subtype which we can't check or src is the same array as dst
2872     // but not necessarily exactly of type default_type.
2873     Label known_ok, halt;
2874 
2875     __ mov_metadata(tmp, default_type->constant_encoding());
2876     if (UseCompressedClassPointers) {
2877       __ encode_klass_not_null(tmp, rscratch1);
2878     }
2879 
2880     if (basic_type != T_OBJECT) {
2881       __ cmp_klass(tmp, dst, tmp2);
2882       __ jcc(Assembler::notEqual, halt);
2883       __ cmp_klass(tmp, src, tmp2);
2884       __ jcc(Assembler::equal, known_ok);
2885     } else {
2886       __ cmp_klass(tmp, dst, tmp2);
2887       __ jcc(Assembler::equal, known_ok);
2888       __ cmpptr(src, dst);
2889       __ jcc(Assembler::equal, known_ok);
2890     }
2891     __ bind(halt);
2892     __ stop("incorrect type information in arraycopy");
2893     __ bind(known_ok);
2894   }
2895 #endif
2896 
2897 #ifndef PRODUCT
2898   if (PrintC1Statistics) {
2899     __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2900   }
2901 #endif
2902 
2903   assert_different_registers(c_rarg0, dst, dst_pos, length);
2904   __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2905   assert_different_registers(c_rarg1, length);
2906   __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2907   __ mov(c_rarg2, length);
2908 
2909   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2910   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2911   const char *name;
2912   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2913   __ call_VM_leaf(entry, 0);
2914 
2915   if (stub != nullptr) {
2916     __ bind(*stub->continuation());
2917   }
2918 }
2919 
2920 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2921   assert(op->crc()->is_single_cpu(),  "crc must be register");
2922   assert(op->val()->is_single_cpu(),  "byte value must be register");
2923   assert(op->result_opr()->is_single_cpu(), "result must be register");
2924   Register crc = op->crc()->as_register();
2925   Register val = op->val()->as_register();
2926   Register res = op->result_opr()->as_register();
2927 
2928   assert_different_registers(val, crc, res);
2929 
2930   __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2931   __ notl(crc); // ~crc
2932   __ update_byte_crc32(crc, val, res);
2933   __ notl(crc); // ~crc
2934   __ mov(res, crc);
2935 }
2936 
2937 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2938   Register obj = op->obj_opr()->as_register();  // may not be an oop
2939   Register hdr = op->hdr_opr()->as_register();
2940   Register lock = op->lock_opr()->as_register();
2941   if (op->code() == lir_lock) {
2942     Register tmp = op->scratch_opr()->as_register();
2943     // add debug info for NullPointerException only if one is possible
2944     int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2945     if (op->info() != nullptr) {
2946       add_debug_info_for_null_check(null_check_offset, op->info());
2947     }
2948     // done
2949   } else if (op->code() == lir_unlock) {
2950     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2951   } else {
2952     Unimplemented();
2953   }
2954   __ bind(*op->stub()->continuation());
2955 }
2956 
2957 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2958   Register obj = op->obj()->as_pointer_register();
2959   Register result = op->result_opr()->as_pointer_register();
2960 
2961   CodeEmitInfo* info = op->info();
2962   if (info != nullptr) {
2963     add_debug_info_for_null_check_here(info);
2964   }
2965 
2966   __ load_klass(result, obj, rscratch1);
2967 }
2968 
2969 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2970   ciMethod* method = op->profiled_method();
2971   int bci          = op->profiled_bci();
2972   ciMethod* callee = op->profiled_callee();
2973   Register tmp_load_klass = rscratch1;
2974 
2975   // Update counter for all call types
2976   ciMethodData* md = method->method_data_or_null();
2977   assert(md != nullptr, "Sanity");
2978   ciProfileData* data = md->bci_to_data(bci);
2979   assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2980   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2981   Register mdo  = op->mdo()->as_register();
2982   __ mov_metadata(mdo, md->constant_encoding());
2983   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2984   // Perform additional virtual call profiling for invokevirtual and
2985   // invokeinterface bytecodes
2986   if (op->should_profile_receiver_type()) {
2987     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2988     Register recv = op->recv()->as_register();
2989     assert_different_registers(mdo, recv);
2990     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2991     ciKlass* known_klass = op->known_holder();
2992     if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2993       // We know the type that will be seen at this call site; we can
2994       // statically update the MethodData* rather than needing to do
2995       // dynamic tests on the receiver type
2996 
2997       // NOTE: we should probably put a lock around this search to
2998       // avoid collisions by concurrent compilations
2999       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3000       uint i;
3001       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3002         ciKlass* receiver = vc_data->receiver(i);
3003         if (known_klass->equals(receiver)) {
3004           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3005           __ addptr(data_addr, DataLayout::counter_increment);
3006           return;
3007         }
3008       }
3009 
3010       // Receiver type not found in profile data; select an empty slot
3011 
3012       // Note that this is less efficient than it should be because it
3013       // always does a write to the receiver part of the
3014       // VirtualCallData rather than just the first time
3015       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3016         ciKlass* receiver = vc_data->receiver(i);
3017         if (receiver == nullptr) {
3018           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3019           __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
3020           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3021           __ addptr(data_addr, DataLayout::counter_increment);
3022           return;
3023         }
3024       }
3025     } else {
3026       __ load_klass(recv, recv, tmp_load_klass);
3027       Label update_done;
3028       type_profile_helper(mdo, md, data, recv, &update_done);
3029       // Receiver did not match any saved receiver and there is no empty row for it.
3030       // Increment total counter to indicate polymorphic case.
3031       __ addptr(counter_addr, DataLayout::counter_increment);
3032 
3033       __ bind(update_done);
3034     }
3035   } else {
3036     // Static call
3037     __ addptr(counter_addr, DataLayout::counter_increment);
3038   }
3039 }
3040 
3041 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3042   Register obj = op->obj()->as_register();
3043   Register tmp = op->tmp()->as_pointer_register();
3044   Register tmp_load_klass = rscratch1;
3045   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3046   ciKlass* exact_klass = op->exact_klass();
3047   intptr_t current_klass = op->current_klass();
3048   bool not_null = op->not_null();
3049   bool no_conflict = op->no_conflict();
3050 
3051   Label update, next, none;
3052 
3053   bool do_null = !not_null;
3054   bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3055   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3056 
3057   assert(do_null || do_update, "why are we here?");
3058   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3059 
3060   __ verify_oop(obj);
3061 
3062 #ifdef ASSERT
3063   if (obj == tmp) {
3064     assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
3065   } else {
3066     assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
3067   }
3068 #endif
3069   if (do_null) {
3070     __ testptr(obj, obj);
3071     __ jccb(Assembler::notZero, update);
3072     if (!TypeEntries::was_null_seen(current_klass)) {
3073       __ testptr(mdo_addr, TypeEntries::null_seen);
3074 #ifndef ASSERT
3075       __ jccb(Assembler::notZero, next); // already set
3076 #else
3077       __ jcc(Assembler::notZero, next); // already set
3078 #endif
3079       // atomic update to prevent overwriting Klass* with 0
3080       __ lock();
3081       __ orptr(mdo_addr, TypeEntries::null_seen);
3082     }
3083     if (do_update) {
3084 #ifndef ASSERT
3085       __ jmpb(next);
3086     }
3087 #else
3088       __ jmp(next);
3089     }
3090   } else {
3091     __ testptr(obj, obj);
3092     __ jcc(Assembler::notZero, update);
3093     __ stop("unexpected null obj");
3094 #endif
3095   }
3096 
3097   __ bind(update);
3098 
3099   if (do_update) {
3100 #ifdef ASSERT
3101     if (exact_klass != nullptr) {
3102       Label ok;
3103       __ load_klass(tmp, obj, tmp_load_klass);
3104       __ push_ppx(tmp);
3105       __ mov_metadata(tmp, exact_klass->constant_encoding());
3106       __ cmpptr(tmp, Address(rsp, 0));
3107       __ jcc(Assembler::equal, ok);
3108       __ stop("exact klass and actual klass differ");
3109       __ bind(ok);
3110       __ pop_ppx(tmp);
3111     }
3112 #endif
3113     if (!no_conflict) {
3114       if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
3115         if (exact_klass != nullptr) {
3116           __ mov_metadata(tmp, exact_klass->constant_encoding());
3117         } else {
3118           __ load_klass(tmp, obj, tmp_load_klass);
3119         }
3120         __ mov(rscratch1, tmp); // save original value before XOR
3121         __ xorptr(tmp, mdo_addr);
3122         __ testptr(tmp, TypeEntries::type_klass_mask);
3123         // klass seen before, nothing to do. The unknown bit may have been
3124         // set already but no need to check.
3125         __ jccb(Assembler::zero, next);
3126 
3127         __ testptr(tmp, TypeEntries::type_unknown);
3128         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3129 
3130         if (TypeEntries::is_type_none(current_klass)) {
3131           __ testptr(mdo_addr, TypeEntries::type_mask);
3132           __ jccb(Assembler::zero, none);
3133           // There is a chance that the checks above (re-reading profiling
3134           // data from memory) fail if another thread has just set the
3135           // profiling to this obj's klass
3136           __ mov(tmp, rscratch1); // get back original value before XOR
3137           __ xorptr(tmp, mdo_addr);
3138           __ testptr(tmp, TypeEntries::type_klass_mask);
3139           __ jccb(Assembler::zero, next);
3140         }
3141       } else {
3142         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3143                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3144 
3145         __ testptr(mdo_addr, TypeEntries::type_unknown);
3146         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3147       }
3148 
3149       // different than before. Cannot keep accurate profile.
3150       __ orptr(mdo_addr, TypeEntries::type_unknown);
3151 
3152       if (TypeEntries::is_type_none(current_klass)) {
3153         __ jmpb(next);
3154 
3155         __ bind(none);
3156         // first time here. Set profile type.
3157         __ movptr(mdo_addr, tmp);
3158 #ifdef ASSERT
3159         __ andptr(tmp, TypeEntries::type_klass_mask);
3160         __ verify_klass_ptr(tmp);
3161 #endif
3162       }
3163     } else {
3164       // There's a single possible klass at this profile point
3165       assert(exact_klass != nullptr, "should be");
3166       if (TypeEntries::is_type_none(current_klass)) {
3167         __ mov_metadata(tmp, exact_klass->constant_encoding());
3168         __ xorptr(tmp, mdo_addr);
3169         __ testptr(tmp, TypeEntries::type_klass_mask);
3170 #ifdef ASSERT
3171         __ jcc(Assembler::zero, next);
3172 
3173         {
3174           Label ok;
3175           __ push_ppx(tmp);
3176           __ testptr(mdo_addr, TypeEntries::type_mask);
3177           __ jcc(Assembler::zero, ok);
3178           // may have been set by another thread
3179           __ mov_metadata(tmp, exact_klass->constant_encoding());
3180           __ xorptr(tmp, mdo_addr);
3181           __ testptr(tmp, TypeEntries::type_mask);
3182           __ jcc(Assembler::zero, ok);
3183 
3184           __ stop("unexpected profiling mismatch");
3185           __ bind(ok);
3186           __ pop_ppx(tmp);
3187         }
3188 #else
3189         __ jccb(Assembler::zero, next);
3190 #endif
3191         // first time here. Set profile type.
3192         __ movptr(mdo_addr, tmp);
3193 #ifdef ASSERT
3194         __ andptr(tmp, TypeEntries::type_klass_mask);
3195         __ verify_klass_ptr(tmp);
3196 #endif
3197       } else {
3198         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3199                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3200 
3201         __ testptr(mdo_addr, TypeEntries::type_unknown);
3202         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3203 
3204         __ orptr(mdo_addr, TypeEntries::type_unknown);
3205       }
3206     }
3207   }
3208   __ bind(next);
3209 }
3210 
3211 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3212   Register obj = op->obj()->as_register();
3213   Register tmp = op->tmp()->as_pointer_register();
3214   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3215   bool not_null = op->not_null();
3216   int flag = op->flag();
3217 
3218   Label not_inline_type;
3219   if (!not_null) {
3220     __ testptr(obj, obj);
3221     __ jccb(Assembler::zero, not_inline_type);
3222   }
3223 
3224   __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3225 
3226   __ orb(mdo_addr, flag);
3227 
3228   __ bind(not_inline_type);
3229 }
3230 
3231 
3232 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3233   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3234 }
3235 
3236 
3237 void LIR_Assembler::align_backward_branch_target() {
3238   __ align(BytesPerWord);
3239 }
3240 
3241 
3242 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3243   if (left->is_single_cpu()) {
3244     __ negl(left->as_register());
3245     move_regs(left->as_register(), dest->as_register());
3246 
3247   } else if (left->is_double_cpu()) {
3248     Register lo = left->as_register_lo();
3249     Register dst = dest->as_register_lo();
3250     __ movptr(dst, lo);
3251     __ negptr(dst);
3252 
3253   } else if (dest->is_single_xmm()) {
3254     assert(!tmp->is_valid(), "do not need temporary");
3255     if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3256       __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3257     }
3258     __ xorps(dest->as_xmm_float_reg(),
3259              ExternalAddress((address)float_signflip_pool),
3260              rscratch1);
3261   } else if (dest->is_double_xmm()) {
3262     assert(!tmp->is_valid(), "do not need temporary");
3263     if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3264       __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3265     }
3266     __ xorpd(dest->as_xmm_double_reg(),
3267              ExternalAddress((address)double_signflip_pool),
3268              rscratch1);
3269   } else {
3270     ShouldNotReachHere();
3271   }
3272 }
3273 
3274 
3275 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3276   assert(src->is_address(), "must be an address");
3277   assert(dest->is_register(), "must be a register");
3278 
3279   PatchingStub* patch = nullptr;
3280   if (patch_code != lir_patch_none) {
3281     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3282   }
3283 
3284   Register reg = dest->as_pointer_register();
3285   LIR_Address* addr = src->as_address_ptr();
3286   __ lea(reg, as_Address(addr));
3287 
3288   if (patch != nullptr) {
3289     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3290   }
3291 }
3292 
3293 
3294 
3295 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3296   assert(!tmp->is_valid(), "don't need temporary");
3297   __ call(RuntimeAddress(dest));
3298   if (info != nullptr) {
3299     add_call_info_here(info);
3300   }
3301   __ post_call_nop();
3302 }
3303 
3304 
3305 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3306   assert(type == T_LONG, "only for volatile long fields");
3307 
3308   if (info != nullptr) {
3309     add_debug_info_for_null_check_here(info);
3310   }
3311 
3312   if (src->is_double_xmm()) {
3313     if (dest->is_double_cpu()) {
3314       __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3315     } else if (dest->is_double_stack()) {
3316       __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3317     } else if (dest->is_address()) {
3318       __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3319     } else {
3320       ShouldNotReachHere();
3321     }
3322 
3323   } else if (dest->is_double_xmm()) {
3324     if (src->is_double_stack()) {
3325       __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3326     } else if (src->is_address()) {
3327       __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3328     } else {
3329       ShouldNotReachHere();
3330     }
3331 
3332   } else {
3333     ShouldNotReachHere();
3334   }
3335 }
3336 
3337 #ifdef ASSERT
3338 // emit run-time assertion
3339 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3340   assert(op->code() == lir_assert, "must be");
3341 
3342   if (op->in_opr1()->is_valid()) {
3343     assert(op->in_opr2()->is_valid(), "both operands must be valid");
3344     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3345   } else {
3346     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3347     assert(op->condition() == lir_cond_always, "no other conditions allowed");
3348   }
3349 
3350   Label ok;
3351   if (op->condition() != lir_cond_always) {
3352     Assembler::Condition acond = Assembler::zero;
3353     switch (op->condition()) {
3354       case lir_cond_equal:        acond = Assembler::equal;       break;
3355       case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
3356       case lir_cond_less:         acond = Assembler::less;        break;
3357       case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
3358       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3359       case lir_cond_greater:      acond = Assembler::greater;     break;
3360       case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
3361       case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
3362       default:                    ShouldNotReachHere();
3363     }
3364     __ jcc(acond, ok);
3365   }
3366   if (op->halt()) {
3367     const char* str = __ code_string(op->msg());
3368     __ stop(str);
3369   } else {
3370     breakpoint();
3371   }
3372   __ bind(ok);
3373 }
3374 #endif
3375 
3376 void LIR_Assembler::membar() {
3377   // QQQ sparc TSO uses this,
3378   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3379 }
3380 
3381 void LIR_Assembler::membar_acquire() {
3382   // No x86 machines currently require load fences
3383 }
3384 
3385 void LIR_Assembler::membar_release() {
3386   // No x86 machines currently require store fences
3387 }
3388 
3389 void LIR_Assembler::membar_loadload() {
3390   // no-op
3391   //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3392 }
3393 
3394 void LIR_Assembler::membar_storestore() {
3395   // no-op
3396   //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3397 }
3398 
3399 void LIR_Assembler::membar_loadstore() {
3400   // no-op
3401   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3402 }
3403 
3404 void LIR_Assembler::membar_storeload() {
3405   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3406 }
3407 
3408 void LIR_Assembler::on_spin_wait() {
3409   __ pause ();
3410 }
3411 
3412 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3413   assert(result_reg->is_register(), "check");
3414   __ mov(result_reg->as_register(), r15_thread);
3415 }
3416 
3417 void LIR_Assembler::check_orig_pc() {
3418   __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3419 }
3420 
3421 void LIR_Assembler::peephole(LIR_List*) {
3422   // do nothing for now
3423 }
3424 
3425 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3426   assert(data == dest, "xchg/xadd uses only 2 operands");
3427 
3428   if (data->type() == T_INT) {
3429     if (code == lir_xadd) {
3430       __ lock();
3431       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3432     } else {
3433       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3434     }
3435   } else if (data->is_oop()) {
3436     assert (code == lir_xchg, "xadd for oops");
3437     Register obj = data->as_register();
3438     if (UseCompressedOops) {
3439       __ encode_heap_oop(obj);
3440       __ xchgl(obj, as_Address(src->as_address_ptr()));
3441       __ decode_heap_oop(obj);
3442     } else {
3443       __ xchgptr(obj, as_Address(src->as_address_ptr()));
3444     }
3445   } else if (data->type() == T_LONG) {
3446     assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3447     if (code == lir_xadd) {
3448       __ lock();
3449       __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3450     } else {
3451       __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3452     }
3453   } else {
3454     ShouldNotReachHere();
3455   }
3456 }
3457 
3458 #undef __