1 /*
   2  * Copyright (c) 2000, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInstance.hpp"
  35 #include "ci/ciUtilities.hpp"
  36 #include "code/SCCache.hpp"
  37 #include "compiler/oopMap.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "gc/shared/gc_globals.hpp"
  40 #include "nativeInst_x86.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "runtime/frame.inline.hpp"
  43 #include "runtime/safepointMechanism.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 #include "utilities/powerOfTwo.hpp"
  47 #include "vmreg_x86.inline.hpp"
  48 
  49 
  50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  51 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  52 // fast versions of NegF/NegD and AbsF/AbsD.
  53 
  54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  55 static address double_quadword(jlong *adr, jlong lo, jlong hi) {
  56   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  57   // of 128-bits operands for SSE instructions.
  58   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  59   // Store the value to a 128-bits operand.
  60   operand[0] = lo;
  61   operand[1] = hi;
  62   return (address)operand;
  63 }
  64 
  65 // Buffer for 128-bits masks used by SSE instructions.
  66 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
  67 
  68 // Static initialization during VM startup.
  69 address LIR_Assembler::float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2],         CONST64(0x7FFFFFFF7FFFFFFF),         CONST64(0x7FFFFFFF7FFFFFFF));
  70 address LIR_Assembler::double_signmask_pool = double_quadword(&fp_signmask_pool[2*2],         CONST64(0x7FFFFFFFFFFFFFFF),         CONST64(0x7FFFFFFFFFFFFFFF));
  71 address LIR_Assembler::float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
  72 address LIR_Assembler::double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
  73 
  74 
  75 NEEDS_CLEANUP // remove this definitions ?
  76 const Register SYNC_header = rax;   // synchronization header
  77 const Register SHIFT_count = rcx;   // where count for shift operations must be
  78 
  79 #define __ _masm->
  80 
  81 
  82 static void select_different_registers(Register preserve,
  83                                        Register extra,
  84                                        Register &tmp1,
  85                                        Register &tmp2) {
  86   if (tmp1 == preserve) {
  87     assert_different_registers(tmp1, tmp2, extra);
  88     tmp1 = extra;
  89   } else if (tmp2 == preserve) {
  90     assert_different_registers(tmp1, tmp2, extra);
  91     tmp2 = extra;
  92   }
  93   assert_different_registers(preserve, tmp1, tmp2);
  94 }
  95 
  96 
  97 
  98 static void select_different_registers(Register preserve,
  99                                        Register extra,
 100                                        Register &tmp1,
 101                                        Register &tmp2,
 102                                        Register &tmp3) {
 103   if (tmp1 == preserve) {
 104     assert_different_registers(tmp1, tmp2, tmp3, extra);
 105     tmp1 = extra;
 106   } else if (tmp2 == preserve) {
 107     assert_different_registers(tmp1, tmp2, tmp3, extra);
 108     tmp2 = extra;
 109   } else if (tmp3 == preserve) {
 110     assert_different_registers(tmp1, tmp2, tmp3, extra);
 111     tmp3 = extra;
 112   }
 113   assert_different_registers(preserve, tmp1, tmp2, tmp3);
 114 }
 115 
 116 
 117 
 118 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
 119   if (opr->is_constant()) {
 120     LIR_Const* constant = opr->as_constant_ptr();
 121     switch (constant->type()) {
 122       case T_INT: {
 123         return true;
 124       }
 125 
 126       default:
 127         return false;
 128     }
 129   }
 130   return false;
 131 }
 132 
 133 
 134 LIR_Opr LIR_Assembler::receiverOpr() {
 135   return FrameMap::receiver_opr;
 136 }
 137 
 138 LIR_Opr LIR_Assembler::osrBufferPointer() {
 139   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 140 }
 141 
 142 //--------------fpu register translations-----------------------
 143 
 144 
 145 address LIR_Assembler::float_constant(float f) {
 146   address const_addr = __ float_constant(f);
 147   if (const_addr == nullptr) {
 148     bailout("const section overflow");
 149     return __ code()->consts()->start();
 150   } else {
 151     return const_addr;
 152   }
 153 }
 154 
 155 
 156 address LIR_Assembler::double_constant(double d) {
 157   address const_addr = __ double_constant(d);
 158   if (const_addr == nullptr) {
 159     bailout("const section overflow");
 160     return __ code()->consts()->start();
 161   } else {
 162     return const_addr;
 163   }
 164 }
 165 
 166 void LIR_Assembler::breakpoint() {
 167   __ int3();
 168 }
 169 
 170 void LIR_Assembler::push(LIR_Opr opr) {
 171   if (opr->is_single_cpu()) {
 172     __ push_reg(opr->as_register());
 173   } else if (opr->is_double_cpu()) {
 174     __ push_reg(opr->as_register_lo());
 175   } else if (opr->is_stack()) {
 176     __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
 177   } else if (opr->is_constant()) {
 178     LIR_Const* const_opr = opr->as_constant_ptr();
 179     if (const_opr->type() == T_OBJECT) {
 180       __ push_oop(const_opr->as_jobject(), rscratch1);
 181     } else if (const_opr->type() == T_INT) {
 182       __ push_jint(const_opr->as_jint());
 183     } else {
 184       ShouldNotReachHere();
 185     }
 186 
 187   } else {
 188     ShouldNotReachHere();
 189   }
 190 }
 191 
 192 void LIR_Assembler::pop(LIR_Opr opr) {
 193   if (opr->is_single_cpu()) {
 194     __ pop_reg(opr->as_register());
 195   } else {
 196     ShouldNotReachHere();
 197   }
 198 }
 199 
 200 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
 201   return addr->base()->is_illegal() && addr->index()->is_illegal();
 202 }
 203 
 204 //-------------------------------------------
 205 
 206 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 207   return as_Address(addr, rscratch1);
 208 }
 209 
 210 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 211   if (addr->base()->is_illegal()) {
 212     assert(addr->index()->is_illegal(), "must be illegal too");
 213     AddressLiteral laddr((address)addr->disp(), relocInfo::none);
 214     if (! __ reachable(laddr)) {
 215       __ movptr(tmp, laddr.addr());
 216       Address res(tmp, 0);
 217       return res;
 218     } else {
 219       return __ as_Address(laddr);
 220     }
 221   }
 222 
 223   Register base = addr->base()->as_pointer_register();
 224 
 225   if (addr->index()->is_illegal()) {
 226     return Address( base, addr->disp());
 227   } else if (addr->index()->is_cpu_register()) {
 228     Register index = addr->index()->as_pointer_register();
 229     return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
 230   } else if (addr->index()->is_constant()) {
 231     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 232     assert(Assembler::is_simm32(addr_offset), "must be");
 233 
 234     return Address(base, addr_offset);
 235   } else {
 236     Unimplemented();
 237     return Address();
 238   }
 239 }
 240 
 241 
 242 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 243   Address base = as_Address(addr);
 244   return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
 245 }
 246 
 247 
 248 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 249   return as_Address(addr);
 250 }
 251 
 252 
 253 void LIR_Assembler::osr_entry() {
 254   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 255   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 256   ValueStack* entry_state = osr_entry->state();
 257   int number_of_locks = entry_state->locks_size();
 258 
 259   // we jump here if osr happens with the interpreter
 260   // state set up to continue at the beginning of the
 261   // loop that triggered osr - in particular, we have
 262   // the following registers setup:
 263   //
 264   // rcx: osr buffer
 265   //
 266 
 267   // build frame
 268   ciMethod* m = compilation()->method();
 269   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 270 
 271   // OSR buffer is
 272   //
 273   // locals[nlocals-1..0]
 274   // monitors[0..number_of_locks]
 275   //
 276   // locals is a direct copy of the interpreter frame so in the osr buffer
 277   // so first slot in the local array is the last local from the interpreter
 278   // and last slot is local[0] (receiver) from the interpreter
 279   //
 280   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 281   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 282   // in the interpreter frame (the method lock if a sync method)
 283 
 284   // Initialize monitors in the compiled activation.
 285   //   rcx: pointer to osr buffer
 286   //
 287   // All other registers are dead at this point and the locals will be
 288   // copied into place by code emitted in the IR.
 289 
 290   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 291   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 292     int monitor_offset = BytesPerWord * method()->max_locals() +
 293       (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
 294     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 295     // the OSR buffer using 2 word entries: first the lock and then
 296     // the oop.
 297     for (int i = 0; i < number_of_locks; i++) {
 298       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 299 #ifdef ASSERT
 300       // verify the interpreter's monitor has a non-null object
 301       {
 302         Label L;
 303         __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
 304         __ jcc(Assembler::notZero, L);
 305         __ stop("locked object is null");
 306         __ bind(L);
 307       }
 308 #endif
 309       __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
 310       __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
 311       __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 312       __ movptr(frame_map()->address_for_monitor_object(i), rbx);
 313     }
 314   }
 315 }
 316 
 317 
 318 // inline cache check; done before the frame is built.
 319 int LIR_Assembler::check_icache() {
 320   return __ ic_check(CodeEntryAlignment);
 321 }
 322 
 323 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 324   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 325   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 326 
 327   Label L_skip_barrier;
 328   Register klass = rscratch1;
 329 
 330   __ mov_metadata(klass, method->holder()->constant_encoding());
 331   __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
 332 
 333   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 334 
 335   __ bind(L_skip_barrier);
 336 }
 337 
 338 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
 339   jobject o = nullptr;
 340   PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
 341   __ movoop(reg, o);
 342   patching_epilog(patch, lir_patch_normal, reg, info);
 343 }
 344 
 345 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 346   Metadata* o = nullptr;
 347   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 348   __ mov_metadata(reg, o);
 349   patching_epilog(patch, lir_patch_normal, reg, info);
 350 }
 351 
 352 // This specifies the rsp decrement needed to build the frame
 353 int LIR_Assembler::initial_frame_size_in_bytes() const {
 354   // if rounding, must let FrameMap know!
 355 
 356   // The frame_map records size in slots (32bit word)
 357 
 358   // subtract two words to account for return address and link
 359   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 360 }
 361 
 362 
 363 int LIR_Assembler::emit_exception_handler() {
 364   // generate code for exception handler
 365   address handler_base = __ start_a_stub(exception_handler_size());
 366   if (handler_base == nullptr) {
 367     // not enough space left for the handler
 368     bailout("exception handler overflow");
 369     return -1;
 370   }
 371 
 372   int offset = code_offset();
 373 
 374   // the exception oop and pc are in rax, and rdx
 375   // no other registers need to be preserved, so invalidate them
 376   __ invalidate_registers(false, true, true, false, true, true);
 377 
 378   // check that there is really an exception
 379   __ verify_not_null_oop(rax);
 380 
 381   // search an exception handler (rax: exception oop, rdx: throwing pc)
 382   __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::handle_exception_from_callee_id)));
 383   __ should_not_reach_here();
 384   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 385   __ end_a_stub();
 386 
 387   return offset;
 388 }
 389 
 390 
 391 // Emit the code to remove the frame from the stack in the exception
 392 // unwind path.
 393 int LIR_Assembler::emit_unwind_handler() {
 394 #ifndef PRODUCT
 395   if (CommentedAssembly) {
 396     _masm->block_comment("Unwind handler");
 397   }
 398 #endif
 399 
 400   int offset = code_offset();
 401 
 402   // Fetch the exception from TLS and clear out exception related thread state
 403   __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
 404   __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
 405   __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
 406 
 407   __ bind(_unwind_handler_entry);
 408   __ verify_not_null_oop(rax);
 409   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 410     __ mov(rbx, rax);  // Preserve the exception (rbx is always callee-saved)
 411   }
 412 
 413   // Perform needed unlocking
 414   MonitorExitStub* stub = nullptr;
 415   if (method()->is_synchronized()) {
 416     monitor_address(0, FrameMap::rax_opr);
 417     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
 418     if (LockingMode == LM_MONITOR) {
 419       __ jmp(*stub->entry());
 420     } else {
 421       __ unlock_object(rdi, rsi, rax, *stub->entry());
 422     }
 423     __ bind(*stub->continuation());
 424   }
 425 
 426   if (compilation()->env()->dtrace_method_probes()) {
 427     __ mov(rdi, r15_thread);
 428     __ mov_metadata(rsi, method()->constant_encoding());
 429     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 430   }
 431 
 432   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 433     __ mov(rax, rbx);  // Restore the exception
 434   }
 435 
 436   // remove the activation and dispatch to the unwind handler
 437   __ remove_frame(initial_frame_size_in_bytes());
 438   __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
 439 
 440   // Emit the slow path assembly
 441   if (stub != nullptr) {
 442     stub->emit_code(this);
 443   }
 444 
 445   return offset;
 446 }
 447 
 448 
 449 int LIR_Assembler::emit_deopt_handler() {
 450   // generate code for exception handler
 451   address handler_base = __ start_a_stub(deopt_handler_size());
 452   if (handler_base == nullptr) {
 453     // not enough space left for the handler
 454     bailout("deopt handler overflow");
 455     return -1;
 456   }
 457 
 458   int offset = code_offset();
 459   InternalAddress here(__ pc());
 460 
 461   __ pushptr(here.addr(), rscratch1);
 462   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 463   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 464   __ end_a_stub();
 465 
 466   return offset;
 467 }
 468 
 469 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 470   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 471   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 472     assert(result->fpu() == 0, "result must already be on TOS");
 473   }
 474 
 475   // Pop the stack before the safepoint code
 476   __ remove_frame(initial_frame_size_in_bytes());
 477 
 478   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 479     __ reserved_stack_check();
 480   }
 481 
 482   // Note: we do not need to round double result; float result has the right precision
 483   // the poll sets the condition code, but no data registers
 484 
 485   code_stub->set_safepoint_offset(__ offset());
 486   __ relocate(relocInfo::poll_return_type);
 487   __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */);
 488   __ ret(0);
 489 }
 490 
 491 
 492 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 493   guarantee(info != nullptr, "Shouldn't be null");
 494   int offset = __ offset();
 495   const Register poll_addr = rscratch1;
 496   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 497   add_debug_info_for_branch(info);
 498   __ relocate(relocInfo::poll_type);
 499   address pre_pc = __ pc();
 500   __ testl(rax, Address(poll_addr, 0));
 501   address post_pc = __ pc();
 502   guarantee(pointer_delta(post_pc, pre_pc, 1) == 3, "must be exact length");
 503   return offset;
 504 }
 505 
 506 
 507 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 508   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 509 }
 510 
 511 void LIR_Assembler::swap_reg(Register a, Register b) {
 512   __ xchgptr(a, b);
 513 }
 514 
 515 
 516 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 517   assert(src->is_constant(), "should not call otherwise");
 518   assert(dest->is_register(), "should not call otherwise");
 519   LIR_Const* c = src->as_constant_ptr();
 520 
 521   switch (c->type()) {
 522     case T_INT: {
 523       assert(patch_code == lir_patch_none, "no patching handled here");
 524       __ movl(dest->as_register(), c->as_jint());
 525       break;
 526     }
 527 
 528     case T_ADDRESS: {
 529       assert(patch_code == lir_patch_none, "no patching handled here");
 530       __ movptr(dest->as_register(), c->as_jint());
 531       break;
 532     }
 533 
 534     case T_LONG: {
 535       assert(patch_code == lir_patch_none, "no patching handled here");
 536       if (SCCache::is_on_for_write()) {
 537         // SCA needs relocation info for card table base
 538         address b = c->as_pointer();
 539         if (is_card_table_address(b)) {
 540           __ lea(dest->as_register_lo(), ExternalAddress(b));
 541           break;
 542         }
 543         if (AOTRuntimeConstants::contains(b)) {
 544           __ load_aotrc_address(dest->as_register_lo(), b);
 545           break;
 546         }
 547       }
 548       __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
 549       break;
 550     }
 551 
 552     case T_OBJECT: {
 553       if (patch_code != lir_patch_none) {
 554         jobject2reg_with_patching(dest->as_register(), info);
 555       } else {
 556         __ movoop(dest->as_register(), c->as_jobject());
 557       }
 558       break;
 559     }
 560 
 561     case T_METADATA: {
 562       if (patch_code != lir_patch_none) {
 563         klass2reg_with_patching(dest->as_register(), info);
 564       } else {
 565         __ mov_metadata(dest->as_register(), c->as_metadata());
 566       }
 567       break;
 568     }
 569 
 570     case T_FLOAT: {
 571       if (dest->is_single_xmm()) {
 572         if (UseAVX <= 2 && c->is_zero_float()) {
 573           __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
 574         } else {
 575           __ movflt(dest->as_xmm_float_reg(),
 576                    InternalAddress(float_constant(c->as_jfloat())));
 577         }
 578       } else {
 579         ShouldNotReachHere();
 580       }
 581       break;
 582     }
 583 
 584     case T_DOUBLE: {
 585       if (dest->is_double_xmm()) {
 586         if (UseAVX <= 2 && c->is_zero_double()) {
 587           __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
 588         } else {
 589           __ movdbl(dest->as_xmm_double_reg(),
 590                     InternalAddress(double_constant(c->as_jdouble())));
 591         }
 592       } else {
 593         ShouldNotReachHere();
 594       }
 595       break;
 596     }
 597 
 598     default:
 599       ShouldNotReachHere();
 600   }
 601 }
 602 
 603 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 604   assert(src->is_constant(), "should not call otherwise");
 605   assert(dest->is_stack(), "should not call otherwise");
 606   LIR_Const* c = src->as_constant_ptr();
 607 
 608   switch (c->type()) {
 609     case T_INT:  // fall through
 610     case T_FLOAT:
 611       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 612       break;
 613 
 614     case T_ADDRESS:
 615       __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 616       break;
 617 
 618     case T_OBJECT:
 619       __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
 620       break;
 621 
 622     case T_LONG:  // fall through
 623     case T_DOUBLE:
 624       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 625                                               lo_word_offset_in_bytes),
 626                 (intptr_t)c->as_jlong_bits(),
 627                 rscratch1);
 628       break;
 629 
 630     default:
 631       ShouldNotReachHere();
 632   }
 633 }
 634 
 635 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 636   assert(src->is_constant(), "should not call otherwise");
 637   assert(dest->is_address(), "should not call otherwise");
 638   LIR_Const* c = src->as_constant_ptr();
 639   LIR_Address* addr = dest->as_address_ptr();
 640 
 641   int null_check_here = code_offset();
 642   switch (type) {
 643     case T_INT:    // fall through
 644     case T_FLOAT:
 645       __ movl(as_Address(addr), c->as_jint_bits());
 646       break;
 647 
 648     case T_ADDRESS:
 649       __ movptr(as_Address(addr), c->as_jint_bits());
 650       break;
 651 
 652     case T_OBJECT:  // fall through
 653     case T_ARRAY:
 654       if (c->as_jobject() == nullptr) {
 655         if (UseCompressedOops && !wide) {
 656           __ movl(as_Address(addr), NULL_WORD);
 657         } else {
 658           __ xorptr(rscratch1, rscratch1);
 659           null_check_here = code_offset();
 660           __ movptr(as_Address(addr), rscratch1);
 661         }
 662       } else {
 663         if (is_literal_address(addr)) {
 664           ShouldNotReachHere();
 665           __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
 666         } else {
 667           __ movoop(rscratch1, c->as_jobject());
 668           if (UseCompressedOops && !wide) {
 669             __ encode_heap_oop(rscratch1);
 670             null_check_here = code_offset();
 671             __ movl(as_Address_lo(addr), rscratch1);
 672           } else {
 673             null_check_here = code_offset();
 674             __ movptr(as_Address_lo(addr), rscratch1);
 675           }
 676         }
 677       }
 678       break;
 679 
 680     case T_LONG:    // fall through
 681     case T_DOUBLE:
 682       if (is_literal_address(addr)) {
 683         ShouldNotReachHere();
 684         __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
 685       } else {
 686         __ movptr(r10, (intptr_t)c->as_jlong_bits());
 687         null_check_here = code_offset();
 688         __ movptr(as_Address_lo(addr), r10);
 689       }
 690       break;
 691 
 692     case T_BOOLEAN: // fall through
 693     case T_BYTE:
 694       __ movb(as_Address(addr), c->as_jint() & 0xFF);
 695       break;
 696 
 697     case T_CHAR:    // fall through
 698     case T_SHORT:
 699       __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
 700       break;
 701 
 702     default:
 703       ShouldNotReachHere();
 704   };
 705 
 706   if (info != nullptr) {
 707     add_debug_info_for_null_check(null_check_here, info);
 708   }
 709 }
 710 
 711 
 712 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 713   assert(src->is_register(), "should not call otherwise");
 714   assert(dest->is_register(), "should not call otherwise");
 715 
 716   // move between cpu-registers
 717   if (dest->is_single_cpu()) {
 718     if (src->type() == T_LONG) {
 719       // Can do LONG -> OBJECT
 720       move_regs(src->as_register_lo(), dest->as_register());
 721       return;
 722     }
 723     assert(src->is_single_cpu(), "must match");
 724     if (src->type() == T_OBJECT) {
 725       __ verify_oop(src->as_register());
 726     }
 727     move_regs(src->as_register(), dest->as_register());
 728 
 729   } else if (dest->is_double_cpu()) {
 730     if (is_reference_type(src->type())) {
 731       // Surprising to me but we can see move of a long to t_object
 732       __ verify_oop(src->as_register());
 733       move_regs(src->as_register(), dest->as_register_lo());
 734       return;
 735     }
 736     assert(src->is_double_cpu(), "must match");
 737     Register f_lo = src->as_register_lo();
 738     Register f_hi = src->as_register_hi();
 739     Register t_lo = dest->as_register_lo();
 740     Register t_hi = dest->as_register_hi();
 741     assert(f_hi == f_lo, "must be same");
 742     assert(t_hi == t_lo, "must be same");
 743     move_regs(f_lo, t_lo);
 744 
 745     // move between xmm-registers
 746   } else if (dest->is_single_xmm()) {
 747     assert(src->is_single_xmm(), "must match");
 748     __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
 749   } else if (dest->is_double_xmm()) {
 750     assert(src->is_double_xmm(), "must match");
 751     __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
 752 
 753   } else {
 754     ShouldNotReachHere();
 755   }
 756 }
 757 
 758 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 759   assert(src->is_register(), "should not call otherwise");
 760   assert(dest->is_stack(), "should not call otherwise");
 761 
 762   if (src->is_single_cpu()) {
 763     Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
 764     if (is_reference_type(type)) {
 765       __ verify_oop(src->as_register());
 766       __ movptr (dst, src->as_register());
 767     } else if (type == T_METADATA || type == T_ADDRESS) {
 768       __ movptr (dst, src->as_register());
 769     } else {
 770       __ movl (dst, src->as_register());
 771     }
 772 
 773   } else if (src->is_double_cpu()) {
 774     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 775     Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
 776     __ movptr (dstLO, src->as_register_lo());
 777 
 778   } else if (src->is_single_xmm()) {
 779     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 780     __ movflt(dst_addr, src->as_xmm_float_reg());
 781 
 782   } else if (src->is_double_xmm()) {
 783     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 784     __ movdbl(dst_addr, src->as_xmm_double_reg());
 785 
 786   } else {
 787     ShouldNotReachHere();
 788   }
 789 }
 790 
 791 
 792 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 793   LIR_Address* to_addr = dest->as_address_ptr();
 794   PatchingStub* patch = nullptr;
 795   Register compressed_src = rscratch1;
 796 
 797   if (is_reference_type(type)) {
 798     __ verify_oop(src->as_register());
 799     if (UseCompressedOops && !wide) {
 800       __ movptr(compressed_src, src->as_register());
 801       __ encode_heap_oop(compressed_src);
 802       if (patch_code != lir_patch_none) {
 803         info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
 804       }
 805     }
 806   }
 807 
 808   if (patch_code != lir_patch_none) {
 809     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 810     Address toa = as_Address(to_addr);
 811     assert(toa.disp() != 0, "must have");
 812   }
 813 
 814   int null_check_here = code_offset();
 815   switch (type) {
 816     case T_FLOAT: {
 817       assert(src->is_single_xmm(), "not a float");
 818       __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
 819       break;
 820     }
 821 
 822     case T_DOUBLE: {
 823       assert(src->is_double_xmm(), "not a double");
 824       __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
 825       break;
 826     }
 827 
 828     case T_ARRAY:   // fall through
 829     case T_OBJECT:  // fall through
 830       if (UseCompressedOops && !wide) {
 831         __ movl(as_Address(to_addr), compressed_src);
 832       } else {
 833         __ movptr(as_Address(to_addr), src->as_register());
 834       }
 835       break;
 836     case T_ADDRESS:
 837       __ movptr(as_Address(to_addr), src->as_register());
 838       break;
 839     case T_INT:
 840       __ movl(as_Address(to_addr), src->as_register());
 841       break;
 842 
 843     case T_LONG: {
 844       Register from_lo = src->as_register_lo();
 845       Register from_hi = src->as_register_hi();
 846       __ movptr(as_Address_lo(to_addr), from_lo);
 847       break;
 848     }
 849 
 850     case T_BYTE:    // fall through
 851     case T_BOOLEAN: {
 852       Register src_reg = src->as_register();
 853       Address dst_addr = as_Address(to_addr);
 854       assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
 855       __ movb(dst_addr, src_reg);
 856       break;
 857     }
 858 
 859     case T_CHAR:    // fall through
 860     case T_SHORT:
 861       __ movw(as_Address(to_addr), src->as_register());
 862       break;
 863 
 864     default:
 865       ShouldNotReachHere();
 866   }
 867   if (info != nullptr) {
 868     add_debug_info_for_null_check(null_check_here, info);
 869   }
 870 
 871   if (patch_code != lir_patch_none) {
 872     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
 873   }
 874 }
 875 
 876 
 877 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 878   assert(src->is_stack(), "should not call otherwise");
 879   assert(dest->is_register(), "should not call otherwise");
 880 
 881   if (dest->is_single_cpu()) {
 882     if (is_reference_type(type)) {
 883       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 884       __ verify_oop(dest->as_register());
 885     } else if (type == T_METADATA || type == T_ADDRESS) {
 886       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 887     } else {
 888       __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 889     }
 890 
 891   } else if (dest->is_double_cpu()) {
 892     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 893     Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
 894     __ movptr(dest->as_register_lo(), src_addr_LO);
 895 
 896   } else if (dest->is_single_xmm()) {
 897     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 898     __ movflt(dest->as_xmm_float_reg(), src_addr);
 899 
 900   } else if (dest->is_double_xmm()) {
 901     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 902     __ movdbl(dest->as_xmm_double_reg(), src_addr);
 903 
 904   } else {
 905     ShouldNotReachHere();
 906   }
 907 }
 908 
 909 
 910 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 911   if (src->is_single_stack()) {
 912     if (is_reference_type(type)) {
 913       __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
 914       __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
 915     } else {
 916       //no pushl on 64bits
 917       __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
 918       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
 919     }
 920 
 921   } else if (src->is_double_stack()) {
 922     __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
 923     __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
 924 
 925   } else {
 926     ShouldNotReachHere();
 927   }
 928 }
 929 
 930 
 931 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 932   assert(src->is_address(), "should not call otherwise");
 933   assert(dest->is_register(), "should not call otherwise");
 934 
 935   LIR_Address* addr = src->as_address_ptr();
 936   Address from_addr = as_Address(addr);
 937 
 938   if (addr->base()->type() == T_OBJECT) {
 939     __ verify_oop(addr->base()->as_pointer_register());
 940   }
 941 
 942   switch (type) {
 943     case T_BOOLEAN: // fall through
 944     case T_BYTE:    // fall through
 945     case T_CHAR:    // fall through
 946     case T_SHORT:
 947       if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
 948         // on pre P6 processors we may get partial register stalls
 949         // so blow away the value of to_rinfo before loading a
 950         // partial word into it.  Do it here so that it precedes
 951         // the potential patch point below.
 952         __ xorptr(dest->as_register(), dest->as_register());
 953       }
 954       break;
 955    default:
 956      break;
 957   }
 958 
 959   PatchingStub* patch = nullptr;
 960   if (patch_code != lir_patch_none) {
 961     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 962     assert(from_addr.disp() != 0, "must have");
 963   }
 964   if (info != nullptr) {
 965     add_debug_info_for_null_check_here(info);
 966   }
 967 
 968   switch (type) {
 969     case T_FLOAT: {
 970       if (dest->is_single_xmm()) {
 971         __ movflt(dest->as_xmm_float_reg(), from_addr);
 972       } else {
 973         ShouldNotReachHere();
 974       }
 975       break;
 976     }
 977 
 978     case T_DOUBLE: {
 979       if (dest->is_double_xmm()) {
 980         __ movdbl(dest->as_xmm_double_reg(), from_addr);
 981       } else {
 982         ShouldNotReachHere();
 983       }
 984       break;
 985     }
 986 
 987     case T_OBJECT:  // fall through
 988     case T_ARRAY:   // fall through
 989       if (UseCompressedOops && !wide) {
 990         __ movl(dest->as_register(), from_addr);
 991       } else {
 992         __ movptr(dest->as_register(), from_addr);
 993       }
 994       break;
 995 
 996     case T_ADDRESS:
 997       __ movptr(dest->as_register(), from_addr);
 998       break;
 999     case T_INT:
1000       __ movl(dest->as_register(), from_addr);
1001       break;
1002 
1003     case T_LONG: {
1004       Register to_lo = dest->as_register_lo();
1005       Register to_hi = dest->as_register_hi();
1006       __ movptr(to_lo, as_Address_lo(addr));
1007       break;
1008     }
1009 
1010     case T_BOOLEAN: // fall through
1011     case T_BYTE: {
1012       Register dest_reg = dest->as_register();
1013       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1014       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1015         __ movsbl(dest_reg, from_addr);
1016       } else {
1017         __ movb(dest_reg, from_addr);
1018         __ shll(dest_reg, 24);
1019         __ sarl(dest_reg, 24);
1020       }
1021       break;
1022     }
1023 
1024     case T_CHAR: {
1025       Register dest_reg = dest->as_register();
1026       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1027       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1028         __ movzwl(dest_reg, from_addr);
1029       } else {
1030         __ movw(dest_reg, from_addr);
1031       }
1032       break;
1033     }
1034 
1035     case T_SHORT: {
1036       Register dest_reg = dest->as_register();
1037       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1038         __ movswl(dest_reg, from_addr);
1039       } else {
1040         __ movw(dest_reg, from_addr);
1041         __ shll(dest_reg, 16);
1042         __ sarl(dest_reg, 16);
1043       }
1044       break;
1045     }
1046 
1047     default:
1048       ShouldNotReachHere();
1049   }
1050 
1051   if (patch != nullptr) {
1052     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1053   }
1054 
1055   if (is_reference_type(type)) {
1056     if (UseCompressedOops && !wide) {
1057       __ decode_heap_oop(dest->as_register());
1058     }
1059 
1060     __ verify_oop(dest->as_register());
1061   }
1062 }
1063 
1064 
1065 NEEDS_CLEANUP; // This could be static?
1066 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1067   int elem_size = type2aelembytes(type);
1068   switch (elem_size) {
1069     case 1: return Address::times_1;
1070     case 2: return Address::times_2;
1071     case 4: return Address::times_4;
1072     case 8: return Address::times_8;
1073   }
1074   ShouldNotReachHere();
1075   return Address::no_scale;
1076 }
1077 
1078 
1079 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1080   switch (op->code()) {
1081     case lir_idiv:
1082     case lir_irem:
1083       arithmetic_idiv(op->code(),
1084                       op->in_opr1(),
1085                       op->in_opr2(),
1086                       op->in_opr3(),
1087                       op->result_opr(),
1088                       op->info());
1089       break;
1090     case lir_fmad:
1091       __ fmad(op->result_opr()->as_xmm_double_reg(),
1092               op->in_opr1()->as_xmm_double_reg(),
1093               op->in_opr2()->as_xmm_double_reg(),
1094               op->in_opr3()->as_xmm_double_reg());
1095       break;
1096     case lir_fmaf:
1097       __ fmaf(op->result_opr()->as_xmm_float_reg(),
1098               op->in_opr1()->as_xmm_float_reg(),
1099               op->in_opr2()->as_xmm_float_reg(),
1100               op->in_opr3()->as_xmm_float_reg());
1101       break;
1102     default:      ShouldNotReachHere(); break;
1103   }
1104 }
1105 
1106 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1107 #ifdef ASSERT
1108   assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1109   if (op->block() != nullptr)  _branch_target_blocks.append(op->block());
1110   if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1111 #endif
1112 
1113   if (op->cond() == lir_cond_always) {
1114     if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1115     __ jmp (*(op->label()));
1116   } else {
1117     Assembler::Condition acond = Assembler::zero;
1118     if (op->code() == lir_cond_float_branch) {
1119       assert(op->ublock() != nullptr, "must have unordered successor");
1120       __ jcc(Assembler::parity, *(op->ublock()->label()));
1121       switch(op->cond()) {
1122         case lir_cond_equal:        acond = Assembler::equal;      break;
1123         case lir_cond_notEqual:     acond = Assembler::notEqual;   break;
1124         case lir_cond_less:         acond = Assembler::below;      break;
1125         case lir_cond_lessEqual:    acond = Assembler::belowEqual; break;
1126         case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1127         case lir_cond_greater:      acond = Assembler::above;      break;
1128         default:                         ShouldNotReachHere();
1129       }
1130     } else {
1131       switch (op->cond()) {
1132         case lir_cond_equal:        acond = Assembler::equal;       break;
1133         case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
1134         case lir_cond_less:         acond = Assembler::less;        break;
1135         case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
1136         case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1137         case lir_cond_greater:      acond = Assembler::greater;     break;
1138         case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
1139         case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
1140         default:                         ShouldNotReachHere();
1141       }
1142     }
1143     __ jcc(acond,*(op->label()));
1144   }
1145 }
1146 
1147 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1148   LIR_Opr src  = op->in_opr();
1149   LIR_Opr dest = op->result_opr();
1150 
1151   switch (op->bytecode()) {
1152     case Bytecodes::_i2l:
1153       __ movl2ptr(dest->as_register_lo(), src->as_register());
1154       break;
1155 
1156     case Bytecodes::_l2i:
1157       __ movl(dest->as_register(), src->as_register_lo());
1158       break;
1159 
1160     case Bytecodes::_i2b:
1161       move_regs(src->as_register(), dest->as_register());
1162       __ sign_extend_byte(dest->as_register());
1163       break;
1164 
1165     case Bytecodes::_i2c:
1166       move_regs(src->as_register(), dest->as_register());
1167       __ andl(dest->as_register(), 0xFFFF);
1168       break;
1169 
1170     case Bytecodes::_i2s:
1171       move_regs(src->as_register(), dest->as_register());
1172       __ sign_extend_short(dest->as_register());
1173       break;
1174 
1175     case Bytecodes::_f2d:
1176       __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1177       break;
1178 
1179     case Bytecodes::_d2f:
1180       __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1181       break;
1182 
1183     case Bytecodes::_i2f:
1184       __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1185       break;
1186 
1187     case Bytecodes::_i2d:
1188       __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1189       break;
1190 
1191     case Bytecodes::_l2f:
1192       __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1193       break;
1194 
1195     case Bytecodes::_l2d:
1196       __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1197       break;
1198 
1199     case Bytecodes::_f2i:
1200       __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1201       break;
1202 
1203     case Bytecodes::_d2i:
1204       __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1205       break;
1206 
1207     case Bytecodes::_f2l:
1208       __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1209       break;
1210 
1211     case Bytecodes::_d2l:
1212       __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1213       break;
1214 
1215     default: ShouldNotReachHere();
1216   }
1217 }
1218 
1219 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1220   if (op->init_check()) {
1221     add_debug_info_for_null_check_here(op->stub()->info());
1222     // init_state needs acquire, but x86 is TSO, and so we are already good.
1223     __ cmpb(Address(op->klass()->as_register(),
1224                     InstanceKlass::init_state_offset()),
1225                     InstanceKlass::fully_initialized);
1226     __ jcc(Assembler::notEqual, *op->stub()->entry());
1227   }
1228   __ allocate_object(op->obj()->as_register(),
1229                      op->tmp1()->as_register(),
1230                      op->tmp2()->as_register(),
1231                      op->header_size(),
1232                      op->object_size(),
1233                      op->klass()->as_register(),
1234                      *op->stub()->entry());
1235   __ bind(*op->stub()->continuation());
1236 }
1237 
1238 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1239   Register len =  op->len()->as_register();
1240   __ movslq(len, len);
1241 
1242   if (UseSlowPath ||
1243       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1244       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1245     __ jmp(*op->stub()->entry());
1246   } else {
1247     Register tmp1 = op->tmp1()->as_register();
1248     Register tmp2 = op->tmp2()->as_register();
1249     Register tmp3 = op->tmp3()->as_register();
1250     if (len == tmp1) {
1251       tmp1 = tmp3;
1252     } else if (len == tmp2) {
1253       tmp2 = tmp3;
1254     } else if (len == tmp3) {
1255       // everything is ok
1256     } else {
1257       __ mov(tmp3, len);
1258     }
1259     __ allocate_array(op->obj()->as_register(),
1260                       len,
1261                       tmp1,
1262                       tmp2,
1263                       arrayOopDesc::base_offset_in_bytes(op->type()),
1264                       array_element_size(op->type()),
1265                       op->klass()->as_register(),
1266                       *op->stub()->entry(),
1267                       op->zero_array());
1268   }
1269   __ bind(*op->stub()->continuation());
1270 }
1271 
1272 void LIR_Assembler::type_profile_helper(Register mdo,
1273                                         ciMethodData *md, ciProfileData *data,
1274                                         Register recv, Label* update_done) {
1275   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1276     Label next_test;
1277     // See if the receiver is receiver[n].
1278     __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1279     __ jccb(Assembler::notEqual, next_test);
1280     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1281     __ addptr(data_addr, DataLayout::counter_increment);
1282     __ jmp(*update_done);
1283     __ bind(next_test);
1284   }
1285 
1286   // Didn't find receiver; find next empty slot and fill it in
1287   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1288     Label next_test;
1289     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1290     __ cmpptr(recv_addr, NULL_WORD);
1291     __ jccb(Assembler::notEqual, next_test);
1292     __ movptr(recv_addr, recv);
1293     __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1294     __ jmp(*update_done);
1295     __ bind(next_test);
1296   }
1297 }
1298 
1299 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1300   // we always need a stub for the failure case.
1301   CodeStub* stub = op->stub();
1302   Register obj = op->object()->as_register();
1303   Register k_RInfo = op->tmp1()->as_register();
1304   Register klass_RInfo = op->tmp2()->as_register();
1305   Register dst = op->result_opr()->as_register();
1306   ciKlass* k = op->klass();
1307   Register Rtmp1 = noreg;
1308   Register tmp_load_klass = rscratch1;
1309 
1310   // check if it needs to be profiled
1311   ciMethodData* md = nullptr;
1312   ciProfileData* data = nullptr;
1313 
1314   if (op->should_profile()) {
1315     ciMethod* method = op->profiled_method();
1316     assert(method != nullptr, "Should have method");
1317     int bci = op->profiled_bci();
1318     md = method->method_data_or_null();
1319     assert(md != nullptr, "Sanity");
1320     data = md->bci_to_data(bci);
1321     assert(data != nullptr,                "need data for type check");
1322     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1323   }
1324   Label* success_target = success;
1325   Label* failure_target = failure;
1326 
1327   if (obj == k_RInfo) {
1328     k_RInfo = dst;
1329   } else if (obj == klass_RInfo) {
1330     klass_RInfo = dst;
1331   }
1332   if (k->is_loaded() && !UseCompressedClassPointers) {
1333     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1334   } else {
1335     Rtmp1 = op->tmp3()->as_register();
1336     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1337   }
1338 
1339   assert_different_registers(obj, k_RInfo, klass_RInfo);
1340 
1341   __ testptr(obj, obj);
1342   if (op->should_profile()) {
1343     Label not_null;
1344     Register mdo  = klass_RInfo;
1345     __ mov_metadata(mdo, md->constant_encoding());
1346     __ jccb(Assembler::notEqual, not_null);
1347     // Object is null; update MDO and exit
1348     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1349     int header_bits = BitData::null_seen_byte_constant();
1350     __ orb(data_addr, header_bits);
1351     __ jmp(*obj_is_null);
1352     __ bind(not_null);
1353 
1354     Label update_done;
1355     Register recv = k_RInfo;
1356     __ load_klass(recv, obj, tmp_load_klass);
1357     type_profile_helper(mdo, md, data, recv, &update_done);
1358 
1359     Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1360     __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1361 
1362     __ bind(update_done);
1363   } else {
1364     __ jcc(Assembler::equal, *obj_is_null);
1365   }
1366 
1367   if (!k->is_loaded()) {
1368     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1369   } else {
1370     __ mov_metadata(k_RInfo, k->constant_encoding());
1371   }
1372   __ verify_oop(obj);
1373 
1374   if (op->fast_check()) {
1375     // get object class
1376     // not a safepoint as obj null check happens earlier
1377     if (UseCompressedClassPointers) {
1378       __ load_klass(Rtmp1, obj, tmp_load_klass);
1379       __ cmpptr(k_RInfo, Rtmp1);
1380     } else {
1381       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1382     }
1383     __ jcc(Assembler::notEqual, *failure_target);
1384     // successful cast, fall through to profile or jump
1385   } else {
1386     // get object class
1387     // not a safepoint as obj null check happens earlier
1388     __ load_klass(klass_RInfo, obj, tmp_load_klass);
1389     if (k->is_loaded()) {
1390       // See if we get an immediate positive hit
1391       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1392       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1393         __ jcc(Assembler::notEqual, *failure_target);
1394         // successful cast, fall through to profile or jump
1395       } else {
1396         // See if we get an immediate positive hit
1397         __ jcc(Assembler::equal, *success_target);
1398         // check for self
1399         __ cmpptr(klass_RInfo, k_RInfo);
1400         __ jcc(Assembler::equal, *success_target);
1401 
1402         __ push(klass_RInfo);
1403         __ push(k_RInfo);
1404         __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1405         __ pop(klass_RInfo);
1406         __ pop(klass_RInfo);
1407         // result is a boolean
1408         __ testl(klass_RInfo, klass_RInfo);
1409         __ jcc(Assembler::equal, *failure_target);
1410         // successful cast, fall through to profile or jump
1411       }
1412     } else {
1413       // perform the fast part of the checking logic
1414       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1415       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1416       __ push(klass_RInfo);
1417       __ push(k_RInfo);
1418       __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1419       __ pop(klass_RInfo);
1420       __ pop(k_RInfo);
1421       // result is a boolean
1422       __ testl(k_RInfo, k_RInfo);
1423       __ jcc(Assembler::equal, *failure_target);
1424       // successful cast, fall through to profile or jump
1425     }
1426   }
1427   __ jmp(*success);
1428 }
1429 
1430 
1431 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1432   Register tmp_load_klass = rscratch1;
1433   LIR_Code code = op->code();
1434   if (code == lir_store_check) {
1435     Register value = op->object()->as_register();
1436     Register array = op->array()->as_register();
1437     Register k_RInfo = op->tmp1()->as_register();
1438     Register klass_RInfo = op->tmp2()->as_register();
1439     Register Rtmp1 = op->tmp3()->as_register();
1440 
1441     CodeStub* stub = op->stub();
1442 
1443     // check if it needs to be profiled
1444     ciMethodData* md = nullptr;
1445     ciProfileData* data = nullptr;
1446 
1447     if (op->should_profile()) {
1448       ciMethod* method = op->profiled_method();
1449       assert(method != nullptr, "Should have method");
1450       int bci = op->profiled_bci();
1451       md = method->method_data_or_null();
1452       assert(md != nullptr, "Sanity");
1453       data = md->bci_to_data(bci);
1454       assert(data != nullptr,                "need data for type check");
1455       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1456     }
1457     Label done;
1458     Label* success_target = &done;
1459     Label* failure_target = stub->entry();
1460 
1461     __ testptr(value, value);
1462     if (op->should_profile()) {
1463       Label not_null;
1464       Register mdo  = klass_RInfo;
1465       __ mov_metadata(mdo, md->constant_encoding());
1466       __ jccb(Assembler::notEqual, not_null);
1467       // Object is null; update MDO and exit
1468       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1469       int header_bits = BitData::null_seen_byte_constant();
1470       __ orb(data_addr, header_bits);
1471       __ jmp(done);
1472       __ bind(not_null);
1473 
1474       Label update_done;
1475       Register recv = k_RInfo;
1476       __ load_klass(recv, value, tmp_load_klass);
1477       type_profile_helper(mdo, md, data, recv, &update_done);
1478 
1479       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1480       __ addptr(counter_addr, DataLayout::counter_increment);
1481       __ bind(update_done);
1482     } else {
1483       __ jcc(Assembler::equal, done);
1484     }
1485 
1486     add_debug_info_for_null_check_here(op->info_for_exception());
1487     __ load_klass(k_RInfo, array, tmp_load_klass);
1488     __ load_klass(klass_RInfo, value, tmp_load_klass);
1489 
1490     // get instance klass (it's already uncompressed)
1491     __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1492     // perform the fast part of the checking logic
1493     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1494     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1495     __ push(klass_RInfo);
1496     __ push(k_RInfo);
1497     __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
1498     __ pop(klass_RInfo);
1499     __ pop(k_RInfo);
1500     // result is a boolean
1501     __ testl(k_RInfo, k_RInfo);
1502     __ jcc(Assembler::equal, *failure_target);
1503     // fall through to the success case
1504 
1505     __ bind(done);
1506   } else
1507     if (code == lir_checkcast) {
1508       Register obj = op->object()->as_register();
1509       Register dst = op->result_opr()->as_register();
1510       Label success;
1511       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1512       __ bind(success);
1513       if (dst != obj) {
1514         __ mov(dst, obj);
1515       }
1516     } else
1517       if (code == lir_instanceof) {
1518         Register obj = op->object()->as_register();
1519         Register dst = op->result_opr()->as_register();
1520         Label success, failure, done;
1521         emit_typecheck_helper(op, &success, &failure, &failure);
1522         __ bind(failure);
1523         __ xorptr(dst, dst);
1524         __ jmpb(done);
1525         __ bind(success);
1526         __ movptr(dst, 1);
1527         __ bind(done);
1528       } else {
1529         ShouldNotReachHere();
1530       }
1531 
1532 }
1533 
1534 
1535 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1536   if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
1537     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1538     Register newval = op->new_value()->as_register();
1539     Register cmpval = op->cmp_value()->as_register();
1540     assert(cmpval == rax, "wrong register");
1541     assert(newval != noreg, "new val must be register");
1542     assert(cmpval != newval, "cmp and new values must be in different registers");
1543     assert(cmpval != addr, "cmp and addr must be in different registers");
1544     assert(newval != addr, "new value and addr must be in different registers");
1545 
1546     if (op->code() == lir_cas_obj) {
1547       if (UseCompressedOops) {
1548         __ encode_heap_oop(cmpval);
1549         __ mov(rscratch1, newval);
1550         __ encode_heap_oop(rscratch1);
1551         __ lock();
1552         // cmpval (rax) is implicitly used by this instruction
1553         __ cmpxchgl(rscratch1, Address(addr, 0));
1554       } else {
1555         __ lock();
1556         __ cmpxchgptr(newval, Address(addr, 0));
1557       }
1558     } else {
1559       assert(op->code() == lir_cas_int, "lir_cas_int expected");
1560       __ lock();
1561       __ cmpxchgl(newval, Address(addr, 0));
1562     }
1563   } else if (op->code() == lir_cas_long) {
1564     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1565     Register newval = op->new_value()->as_register_lo();
1566     Register cmpval = op->cmp_value()->as_register_lo();
1567     assert(cmpval == rax, "wrong register");
1568     assert(newval != noreg, "new val must be register");
1569     assert(cmpval != newval, "cmp and new values must be in different registers");
1570     assert(cmpval != addr, "cmp and addr must be in different registers");
1571     assert(newval != addr, "new value and addr must be in different registers");
1572     __ lock();
1573     __ cmpxchgq(newval, Address(addr, 0));
1574   } else {
1575     Unimplemented();
1576   }
1577 }
1578 
1579 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1580                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1581   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1582 
1583   Assembler::Condition acond, ncond;
1584   switch (condition) {
1585     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1586     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1587     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1588     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1589     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1590     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1591     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1592     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1593     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1594                                 ShouldNotReachHere();
1595   }
1596 
1597   if (opr1->is_cpu_register()) {
1598     reg2reg(opr1, result);
1599   } else if (opr1->is_stack()) {
1600     stack2reg(opr1, result, result->type());
1601   } else if (opr1->is_constant()) {
1602     const2reg(opr1, result, lir_patch_none, nullptr);
1603   } else {
1604     ShouldNotReachHere();
1605   }
1606 
1607   if (VM_Version::supports_cmov() && !opr2->is_constant()) {
1608     // optimized version that does not require a branch
1609     if (opr2->is_single_cpu()) {
1610       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
1611       __ cmov(ncond, result->as_register(), opr2->as_register());
1612     } else if (opr2->is_double_cpu()) {
1613       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1614       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
1615       __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
1616     } else if (opr2->is_single_stack()) {
1617       __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
1618     } else if (opr2->is_double_stack()) {
1619       __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
1620     } else {
1621       ShouldNotReachHere();
1622     }
1623 
1624   } else {
1625     Label skip;
1626     __ jccb(acond, skip);
1627     if (opr2->is_cpu_register()) {
1628       reg2reg(opr2, result);
1629     } else if (opr2->is_stack()) {
1630       stack2reg(opr2, result, result->type());
1631     } else if (opr2->is_constant()) {
1632       const2reg(opr2, result, lir_patch_none, nullptr);
1633     } else {
1634       ShouldNotReachHere();
1635     }
1636     __ bind(skip);
1637   }
1638 }
1639 
1640 
1641 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info) {
1642   assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1643 
1644   if (left->is_single_cpu()) {
1645     assert(left == dest, "left and dest must be equal");
1646     Register lreg = left->as_register();
1647 
1648     if (right->is_single_cpu()) {
1649       // cpu register - cpu register
1650       Register rreg = right->as_register();
1651       switch (code) {
1652         case lir_add: __ addl (lreg, rreg); break;
1653         case lir_sub: __ subl (lreg, rreg); break;
1654         case lir_mul: __ imull(lreg, rreg); break;
1655         default:      ShouldNotReachHere();
1656       }
1657 
1658     } else if (right->is_stack()) {
1659       // cpu register - stack
1660       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1661       switch (code) {
1662         case lir_add: __ addl(lreg, raddr); break;
1663         case lir_sub: __ subl(lreg, raddr); break;
1664         default:      ShouldNotReachHere();
1665       }
1666 
1667     } else if (right->is_constant()) {
1668       // cpu register - constant
1669       jint c = right->as_constant_ptr()->as_jint();
1670       switch (code) {
1671         case lir_add: {
1672           __ incrementl(lreg, c);
1673           break;
1674         }
1675         case lir_sub: {
1676           __ decrementl(lreg, c);
1677           break;
1678         }
1679         default: ShouldNotReachHere();
1680       }
1681 
1682     } else {
1683       ShouldNotReachHere();
1684     }
1685 
1686   } else if (left->is_double_cpu()) {
1687     assert(left == dest, "left and dest must be equal");
1688     Register lreg_lo = left->as_register_lo();
1689     Register lreg_hi = left->as_register_hi();
1690 
1691     if (right->is_double_cpu()) {
1692       // cpu register - cpu register
1693       Register rreg_lo = right->as_register_lo();
1694       Register rreg_hi = right->as_register_hi();
1695       assert_different_registers(lreg_lo, rreg_lo);
1696       switch (code) {
1697         case lir_add:
1698           __ addptr(lreg_lo, rreg_lo);
1699           break;
1700         case lir_sub:
1701           __ subptr(lreg_lo, rreg_lo);
1702           break;
1703         case lir_mul:
1704           __ imulq(lreg_lo, rreg_lo);
1705           break;
1706         default:
1707           ShouldNotReachHere();
1708       }
1709 
1710     } else if (right->is_constant()) {
1711       // cpu register - constant
1712       jlong c = right->as_constant_ptr()->as_jlong_bits();
1713       __ movptr(r10, (intptr_t) c);
1714       switch (code) {
1715         case lir_add:
1716           __ addptr(lreg_lo, r10);
1717           break;
1718         case lir_sub:
1719           __ subptr(lreg_lo, r10);
1720           break;
1721         default:
1722           ShouldNotReachHere();
1723       }
1724 
1725     } else {
1726       ShouldNotReachHere();
1727     }
1728 
1729   } else if (left->is_single_xmm()) {
1730     assert(left == dest, "left and dest must be equal");
1731     XMMRegister lreg = left->as_xmm_float_reg();
1732 
1733     if (right->is_single_xmm()) {
1734       XMMRegister rreg = right->as_xmm_float_reg();
1735       switch (code) {
1736         case lir_add: __ addss(lreg, rreg);  break;
1737         case lir_sub: __ subss(lreg, rreg);  break;
1738         case lir_mul: __ mulss(lreg, rreg);  break;
1739         case lir_div: __ divss(lreg, rreg);  break;
1740         default: ShouldNotReachHere();
1741       }
1742     } else {
1743       Address raddr;
1744       if (right->is_single_stack()) {
1745         raddr = frame_map()->address_for_slot(right->single_stack_ix());
1746       } else if (right->is_constant()) {
1747         // hack for now
1748         raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
1749       } else {
1750         ShouldNotReachHere();
1751       }
1752       switch (code) {
1753         case lir_add: __ addss(lreg, raddr);  break;
1754         case lir_sub: __ subss(lreg, raddr);  break;
1755         case lir_mul: __ mulss(lreg, raddr);  break;
1756         case lir_div: __ divss(lreg, raddr);  break;
1757         default: ShouldNotReachHere();
1758       }
1759     }
1760 
1761   } else if (left->is_double_xmm()) {
1762     assert(left == dest, "left and dest must be equal");
1763 
1764     XMMRegister lreg = left->as_xmm_double_reg();
1765     if (right->is_double_xmm()) {
1766       XMMRegister rreg = right->as_xmm_double_reg();
1767       switch (code) {
1768         case lir_add: __ addsd(lreg, rreg);  break;
1769         case lir_sub: __ subsd(lreg, rreg);  break;
1770         case lir_mul: __ mulsd(lreg, rreg);  break;
1771         case lir_div: __ divsd(lreg, rreg);  break;
1772         default: ShouldNotReachHere();
1773       }
1774     } else {
1775       Address raddr;
1776       if (right->is_double_stack()) {
1777         raddr = frame_map()->address_for_slot(right->double_stack_ix());
1778       } else if (right->is_constant()) {
1779         // hack for now
1780         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
1781       } else {
1782         ShouldNotReachHere();
1783       }
1784       switch (code) {
1785         case lir_add: __ addsd(lreg, raddr);  break;
1786         case lir_sub: __ subsd(lreg, raddr);  break;
1787         case lir_mul: __ mulsd(lreg, raddr);  break;
1788         case lir_div: __ divsd(lreg, raddr);  break;
1789         default: ShouldNotReachHere();
1790       }
1791     }
1792 
1793   } else if (left->is_single_stack() || left->is_address()) {
1794     assert(left == dest, "left and dest must be equal");
1795 
1796     Address laddr;
1797     if (left->is_single_stack()) {
1798       laddr = frame_map()->address_for_slot(left->single_stack_ix());
1799     } else if (left->is_address()) {
1800       laddr = as_Address(left->as_address_ptr());
1801     } else {
1802       ShouldNotReachHere();
1803     }
1804 
1805     if (right->is_single_cpu()) {
1806       Register rreg = right->as_register();
1807       switch (code) {
1808         case lir_add: __ addl(laddr, rreg); break;
1809         case lir_sub: __ subl(laddr, rreg); break;
1810         default:      ShouldNotReachHere();
1811       }
1812     } else if (right->is_constant()) {
1813       jint c = right->as_constant_ptr()->as_jint();
1814       switch (code) {
1815         case lir_add: {
1816           __ incrementl(laddr, c);
1817           break;
1818         }
1819         case lir_sub: {
1820           __ decrementl(laddr, c);
1821           break;
1822         }
1823         default: ShouldNotReachHere();
1824       }
1825     } else {
1826       ShouldNotReachHere();
1827     }
1828 
1829   } else {
1830     ShouldNotReachHere();
1831   }
1832 }
1833 
1834 
1835 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1836   if (value->is_double_xmm()) {
1837     switch(code) {
1838       case lir_abs :
1839         {
1840           if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
1841             __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
1842           }
1843           assert(!tmp->is_valid(), "do not need temporary");
1844           __ andpd(dest->as_xmm_double_reg(),
1845                    ExternalAddress(LIR_Assembler::double_signmask_pool),
1846                    rscratch1);
1847         }
1848         break;
1849 
1850       case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
1851       // all other intrinsics are not available in the SSE instruction set, so FPU is used
1852       default      : ShouldNotReachHere();
1853     }
1854 
1855   } else if (code == lir_f2hf) {
1856     __ flt_to_flt16(dest->as_register(), value->as_xmm_float_reg(), tmp->as_xmm_float_reg());
1857   } else if (code == lir_hf2f) {
1858     __ flt16_to_flt(dest->as_xmm_float_reg(), value->as_register());
1859   } else {
1860     Unimplemented();
1861   }
1862 }
1863 
1864 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1865   // assert(left->destroys_register(), "check");
1866   if (left->is_single_cpu()) {
1867     Register reg = left->as_register();
1868     if (right->is_constant()) {
1869       int val = right->as_constant_ptr()->as_jint();
1870       switch (code) {
1871         case lir_logic_and: __ andl (reg, val); break;
1872         case lir_logic_or:  __ orl  (reg, val); break;
1873         case lir_logic_xor: __ xorl (reg, val); break;
1874         default: ShouldNotReachHere();
1875       }
1876     } else if (right->is_stack()) {
1877       // added support for stack operands
1878       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
1879       switch (code) {
1880         case lir_logic_and: __ andl (reg, raddr); break;
1881         case lir_logic_or:  __ orl  (reg, raddr); break;
1882         case lir_logic_xor: __ xorl (reg, raddr); break;
1883         default: ShouldNotReachHere();
1884       }
1885     } else {
1886       Register rright = right->as_register();
1887       switch (code) {
1888         case lir_logic_and: __ andptr (reg, rright); break;
1889         case lir_logic_or : __ orptr  (reg, rright); break;
1890         case lir_logic_xor: __ xorptr (reg, rright); break;
1891         default: ShouldNotReachHere();
1892       }
1893     }
1894     move_regs(reg, dst->as_register());
1895   } else {
1896     Register l_lo = left->as_register_lo();
1897     Register l_hi = left->as_register_hi();
1898     if (right->is_constant()) {
1899       __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
1900       switch (code) {
1901         case lir_logic_and:
1902           __ andq(l_lo, rscratch1);
1903           break;
1904         case lir_logic_or:
1905           __ orq(l_lo, rscratch1);
1906           break;
1907         case lir_logic_xor:
1908           __ xorq(l_lo, rscratch1);
1909           break;
1910         default: ShouldNotReachHere();
1911       }
1912     } else {
1913       Register r_lo;
1914       if (is_reference_type(right->type())) {
1915         r_lo = right->as_register();
1916       } else {
1917         r_lo = right->as_register_lo();
1918       }
1919       switch (code) {
1920         case lir_logic_and:
1921           __ andptr(l_lo, r_lo);
1922           break;
1923         case lir_logic_or:
1924           __ orptr(l_lo, r_lo);
1925           break;
1926         case lir_logic_xor:
1927           __ xorptr(l_lo, r_lo);
1928           break;
1929         default: ShouldNotReachHere();
1930       }
1931     }
1932 
1933     Register dst_lo = dst->as_register_lo();
1934     Register dst_hi = dst->as_register_hi();
1935 
1936     move_regs(l_lo, dst_lo);
1937   }
1938 }
1939 
1940 
1941 // we assume that rax, and rdx can be overwritten
1942 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
1943 
1944   assert(left->is_single_cpu(),   "left must be register");
1945   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
1946   assert(result->is_single_cpu(), "result must be register");
1947 
1948   //  assert(left->destroys_register(), "check");
1949   //  assert(right->destroys_register(), "check");
1950 
1951   Register lreg = left->as_register();
1952   Register dreg = result->as_register();
1953 
1954   if (right->is_constant()) {
1955     jint divisor = right->as_constant_ptr()->as_jint();
1956     assert(divisor > 0 && is_power_of_2(divisor), "must be");
1957     if (code == lir_idiv) {
1958       assert(lreg == rax, "must be rax,");
1959       assert(temp->as_register() == rdx, "tmp register must be rdx");
1960       __ cdql(); // sign extend into rdx:rax
1961       if (divisor == 2) {
1962         __ subl(lreg, rdx);
1963       } else {
1964         __ andl(rdx, divisor - 1);
1965         __ addl(lreg, rdx);
1966       }
1967       __ sarl(lreg, log2i_exact(divisor));
1968       move_regs(lreg, dreg);
1969     } else if (code == lir_irem) {
1970       Label done;
1971       __ mov(dreg, lreg);
1972       __ andl(dreg, 0x80000000 | (divisor - 1));
1973       __ jcc(Assembler::positive, done);
1974       __ decrement(dreg);
1975       __ orl(dreg, ~(divisor - 1));
1976       __ increment(dreg);
1977       __ bind(done);
1978     } else {
1979       ShouldNotReachHere();
1980     }
1981   } else {
1982     Register rreg = right->as_register();
1983     assert(lreg == rax, "left register must be rax,");
1984     assert(rreg != rdx, "right register must not be rdx");
1985     assert(temp->as_register() == rdx, "tmp register must be rdx");
1986 
1987     move_regs(lreg, rax);
1988 
1989     int idivl_offset = __ corrected_idivl(rreg);
1990     if (ImplicitDiv0Checks) {
1991       add_debug_info_for_div0(idivl_offset, info);
1992     }
1993     if (code == lir_irem) {
1994       move_regs(rdx, dreg); // result is in rdx
1995     } else {
1996       move_regs(rax, dreg);
1997     }
1998   }
1999 }
2000 
2001 
2002 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2003   if (opr1->is_single_cpu()) {
2004     Register reg1 = opr1->as_register();
2005     if (opr2->is_single_cpu()) {
2006       // cpu register - cpu register
2007       if (is_reference_type(opr1->type())) {
2008         __ cmpoop(reg1, opr2->as_register());
2009       } else {
2010         assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2011         __ cmpl(reg1, opr2->as_register());
2012       }
2013     } else if (opr2->is_stack()) {
2014       // cpu register - stack
2015       if (is_reference_type(opr1->type())) {
2016         __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2017       } else {
2018         __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2019       }
2020     } else if (opr2->is_constant()) {
2021       // cpu register - constant
2022       LIR_Const* c = opr2->as_constant_ptr();
2023       if (c->type() == T_INT) {
2024         jint i = c->as_jint();
2025         if (i == 0) {
2026           __ testl(reg1, reg1);
2027         } else {
2028           __ cmpl(reg1, i);
2029         }
2030       } else if (c->type() == T_METADATA) {
2031         // All we need for now is a comparison with null for equality.
2032         assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2033         Metadata* m = c->as_metadata();
2034         if (m == nullptr) {
2035           __ testptr(reg1, reg1);
2036         } else {
2037           ShouldNotReachHere();
2038         }
2039       } else if (is_reference_type(c->type())) {
2040         // In 64bit oops are single register
2041         jobject o = c->as_jobject();
2042         if (o == nullptr) {
2043           __ testptr(reg1, reg1);
2044         } else {
2045           __ cmpoop(reg1, o, rscratch1);
2046         }
2047       } else {
2048         fatal("unexpected type: %s", basictype_to_str(c->type()));
2049       }
2050       // cpu register - address
2051     } else if (opr2->is_address()) {
2052       if (op->info() != nullptr) {
2053         add_debug_info_for_null_check_here(op->info());
2054       }
2055       __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2056     } else {
2057       ShouldNotReachHere();
2058     }
2059 
2060   } else if(opr1->is_double_cpu()) {
2061     Register xlo = opr1->as_register_lo();
2062     Register xhi = opr1->as_register_hi();
2063     if (opr2->is_double_cpu()) {
2064       __ cmpptr(xlo, opr2->as_register_lo());
2065     } else if (opr2->is_constant()) {
2066       // cpu register - constant 0
2067       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2068       __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2069     } else {
2070       ShouldNotReachHere();
2071     }
2072 
2073   } else if (opr1->is_single_xmm()) {
2074     XMMRegister reg1 = opr1->as_xmm_float_reg();
2075     if (opr2->is_single_xmm()) {
2076       // xmm register - xmm register
2077       __ ucomiss(reg1, opr2->as_xmm_float_reg());
2078     } else if (opr2->is_stack()) {
2079       // xmm register - stack
2080       __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2081     } else if (opr2->is_constant()) {
2082       // xmm register - constant
2083       __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2084     } else if (opr2->is_address()) {
2085       // xmm register - address
2086       if (op->info() != nullptr) {
2087         add_debug_info_for_null_check_here(op->info());
2088       }
2089       __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2090     } else {
2091       ShouldNotReachHere();
2092     }
2093 
2094   } else if (opr1->is_double_xmm()) {
2095     XMMRegister reg1 = opr1->as_xmm_double_reg();
2096     if (opr2->is_double_xmm()) {
2097       // xmm register - xmm register
2098       __ ucomisd(reg1, opr2->as_xmm_double_reg());
2099     } else if (opr2->is_stack()) {
2100       // xmm register - stack
2101       __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2102     } else if (opr2->is_constant()) {
2103       // xmm register - constant
2104       __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2105     } else if (opr2->is_address()) {
2106       // xmm register - address
2107       if (op->info() != nullptr) {
2108         add_debug_info_for_null_check_here(op->info());
2109       }
2110       __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2111     } else {
2112       ShouldNotReachHere();
2113     }
2114 
2115   } else if (opr1->is_address() && opr2->is_constant()) {
2116     LIR_Const* c = opr2->as_constant_ptr();
2117     if (is_reference_type(c->type())) {
2118       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2119       __ movoop(rscratch1, c->as_jobject());
2120     }
2121     if (op->info() != nullptr) {
2122       add_debug_info_for_null_check_here(op->info());
2123     }
2124     // special case: address - constant
2125     LIR_Address* addr = opr1->as_address_ptr();
2126     if (c->type() == T_INT) {
2127       __ cmpl(as_Address(addr), c->as_jint());
2128     } else if (is_reference_type(c->type())) {
2129       // %%% Make this explode if addr isn't reachable until we figure out a
2130       // better strategy by giving noreg as the temp for as_Address
2131       __ cmpoop(rscratch1, as_Address(addr, noreg));
2132     } else {
2133       ShouldNotReachHere();
2134     }
2135 
2136   } else {
2137     ShouldNotReachHere();
2138   }
2139 }
2140 
2141 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2142   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2143     if (left->is_single_xmm()) {
2144       assert(right->is_single_xmm(), "must match");
2145       __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2146     } else if (left->is_double_xmm()) {
2147       assert(right->is_double_xmm(), "must match");
2148       __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2149 
2150     } else {
2151       ShouldNotReachHere();
2152     }
2153   } else {
2154     assert(code == lir_cmp_l2i, "check");
2155     Label done;
2156     Register dest = dst->as_register();
2157     __ cmpptr(left->as_register_lo(), right->as_register_lo());
2158     __ movl(dest, -1);
2159     __ jccb(Assembler::less, done);
2160     __ setb(Assembler::notZero, dest);
2161     __ movzbl(dest, dest);
2162     __ bind(done);
2163   }
2164 }
2165 
2166 
2167 void LIR_Assembler::align_call(LIR_Code code) {
2168   // make sure that the displacement word of the call ends up word aligned
2169   int offset = __ offset();
2170   switch (code) {
2171   case lir_static_call:
2172   case lir_optvirtual_call:
2173   case lir_dynamic_call:
2174     offset += NativeCall::displacement_offset;
2175     break;
2176   case lir_icvirtual_call:
2177     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2178     break;
2179   default: ShouldNotReachHere();
2180   }
2181   __ align(BytesPerWord, offset);
2182 }
2183 
2184 
2185 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2186   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2187          "must be aligned");
2188   __ call(AddressLiteral(op->addr(), rtype));
2189   add_call_info(code_offset(), op->info());
2190   __ post_call_nop();
2191 }
2192 
2193 
2194 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2195   __ ic_call(op->addr());
2196   add_call_info(code_offset(), op->info());
2197   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2198          "must be aligned");
2199   __ post_call_nop();
2200 }
2201 
2202 
2203 void LIR_Assembler::emit_static_call_stub() {
2204   address call_pc = __ pc();
2205   address stub = __ start_a_stub(call_stub_size());
2206   if (stub == nullptr) {
2207     bailout("static call stub overflow");
2208     return;
2209   }
2210 
2211   int start = __ offset();
2212 
2213   // make sure that the displacement word of the call ends up word aligned
2214   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2215   __ relocate(static_stub_Relocation::spec(call_pc));
2216   __ mov_metadata(rbx, (Metadata*)nullptr);
2217   // must be set to -1 at code generation time
2218   assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
2219   // On 64bit this will die since it will take a movq & jmp, must be only a jmp
2220   __ jump(RuntimeAddress(__ pc()));
2221 
2222   assert(__ offset() - start <= call_stub_size(), "stub too big");
2223   __ end_a_stub();
2224 }
2225 
2226 
2227 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2228   assert(exceptionOop->as_register() == rax, "must match");
2229   assert(exceptionPC->as_register() == rdx, "must match");
2230 
2231   // exception object is not added to oop map by LinearScan
2232   // (LinearScan assumes that no oops are in fixed registers)
2233   info->add_register_oop(exceptionOop);
2234   C1StubId unwind_id;
2235 
2236   // get current pc information
2237   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2238   int pc_for_athrow_offset = __ offset();
2239   InternalAddress pc_for_athrow(__ pc());
2240   __ lea(exceptionPC->as_register(), pc_for_athrow);
2241   add_call_info(pc_for_athrow_offset, info); // for exception handler
2242 
2243   __ verify_not_null_oop(rax);
2244   // search an exception handler (rax: exception oop, rdx: throwing pc)
2245   if (compilation()->has_fpu_code()) {
2246     unwind_id = C1StubId::handle_exception_id;
2247   } else {
2248     unwind_id = C1StubId::handle_exception_nofpu_id;
2249   }
2250   __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2251 
2252   // enough room for two byte trap
2253   __ nop();
2254 }
2255 
2256 
2257 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2258   assert(exceptionOop->as_register() == rax, "must match");
2259 
2260   __ jmp(_unwind_handler_entry);
2261 }
2262 
2263 
2264 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2265 
2266   // optimized version for linear scan:
2267   // * count must be already in ECX (guaranteed by LinearScan)
2268   // * left and dest must be equal
2269   // * tmp must be unused
2270   assert(count->as_register() == SHIFT_count, "count must be in ECX");
2271   assert(left == dest, "left and dest must be equal");
2272   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2273 
2274   if (left->is_single_cpu()) {
2275     Register value = left->as_register();
2276     assert(value != SHIFT_count, "left cannot be ECX");
2277 
2278     switch (code) {
2279       case lir_shl:  __ shll(value); break;
2280       case lir_shr:  __ sarl(value); break;
2281       case lir_ushr: __ shrl(value); break;
2282       default: ShouldNotReachHere();
2283     }
2284   } else if (left->is_double_cpu()) {
2285     Register lo = left->as_register_lo();
2286     Register hi = left->as_register_hi();
2287     assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
2288     switch (code) {
2289       case lir_shl:  __ shlptr(lo);        break;
2290       case lir_shr:  __ sarptr(lo);        break;
2291       case lir_ushr: __ shrptr(lo);        break;
2292       default: ShouldNotReachHere();
2293     }
2294   } else {
2295     ShouldNotReachHere();
2296   }
2297 }
2298 
2299 
2300 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2301   if (dest->is_single_cpu()) {
2302     // first move left into dest so that left is not destroyed by the shift
2303     Register value = dest->as_register();
2304     count = count & 0x1F; // Java spec
2305 
2306     move_regs(left->as_register(), value);
2307     switch (code) {
2308       case lir_shl:  __ shll(value, count); break;
2309       case lir_shr:  __ sarl(value, count); break;
2310       case lir_ushr: __ shrl(value, count); break;
2311       default: ShouldNotReachHere();
2312     }
2313   } else if (dest->is_double_cpu()) {
2314     // first move left into dest so that left is not destroyed by the shift
2315     Register value = dest->as_register_lo();
2316     count = count & 0x1F; // Java spec
2317 
2318     move_regs(left->as_register_lo(), value);
2319     switch (code) {
2320       case lir_shl:  __ shlptr(value, count); break;
2321       case lir_shr:  __ sarptr(value, count); break;
2322       case lir_ushr: __ shrptr(value, count); break;
2323       default: ShouldNotReachHere();
2324     }
2325   } else {
2326     ShouldNotReachHere();
2327   }
2328 }
2329 
2330 
2331 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2332   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2333   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2334   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2335   __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
2336 }
2337 
2338 
2339 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2340   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2341   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2342   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2343   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2344 }
2345 
2346 
2347 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2348   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2349   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2350   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2351   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2352 }
2353 
2354 
2355 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2356   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2357   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2358   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2359   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2360 }
2361 
2362 
2363 // This code replaces a call to arraycopy; no exception may
2364 // be thrown in this code, they must be thrown in the System.arraycopy
2365 // activation frame; we could save some checks if this would not be the case
2366 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2367   ciArrayKlass* default_type = op->expected_type();
2368   Register src = op->src()->as_register();
2369   Register dst = op->dst()->as_register();
2370   Register src_pos = op->src_pos()->as_register();
2371   Register dst_pos = op->dst_pos()->as_register();
2372   Register length  = op->length()->as_register();
2373   Register tmp = op->tmp()->as_register();
2374   Register tmp_load_klass = rscratch1;
2375   Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2376 
2377   CodeStub* stub = op->stub();
2378   int flags = op->flags();
2379   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2380   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2381 
2382   // if we don't know anything, just go through the generic arraycopy
2383   if (default_type == nullptr) {
2384     // save outgoing arguments on stack in case call to System.arraycopy is needed
2385     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2386     // for interpreter calling conventions. Now we have to do it in new style conventions.
2387     // For the moment until C1 gets the new register allocator I just force all the
2388     // args to the right place (except the register args) and then on the back side
2389     // reload the register args properly if we go slow path. Yuck
2390 
2391     // These are proper for the calling convention
2392     store_parameter(length, 2);
2393     store_parameter(dst_pos, 1);
2394     store_parameter(dst, 0);
2395 
2396     // these are just temporary placements until we need to reload
2397     store_parameter(src_pos, 3);
2398     store_parameter(src, 4);
2399 
2400     address copyfunc_addr = StubRoutines::generic_arraycopy();
2401     assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2402 
2403     // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
2404     // The arguments are in java calling convention so we can trivially shift them to C
2405     // convention
2406     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2407     __ mov(c_rarg0, j_rarg0);
2408     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2409     __ mov(c_rarg1, j_rarg1);
2410     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2411     __ mov(c_rarg2, j_rarg2);
2412     assert_different_registers(c_rarg3, j_rarg4);
2413     __ mov(c_rarg3, j_rarg3);
2414 #ifdef _WIN64
2415     // Allocate abi space for args but be sure to keep stack aligned
2416     __ subptr(rsp, 6*wordSize);
2417     store_parameter(j_rarg4, 4);
2418 #ifndef PRODUCT
2419     if (PrintC1Statistics) {
2420       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2421     }
2422 #endif
2423     __ call(RuntimeAddress(copyfunc_addr));
2424     __ addptr(rsp, 6*wordSize);
2425 #else
2426     __ mov(c_rarg4, j_rarg4);
2427 #ifndef PRODUCT
2428     if (PrintC1Statistics) {
2429       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
2430     }
2431 #endif
2432     __ call(RuntimeAddress(copyfunc_addr));
2433 #endif // _WIN64
2434 
2435     __ testl(rax, rax);
2436     __ jcc(Assembler::equal, *stub->continuation());
2437 
2438     __ mov(tmp, rax);
2439     __ xorl(tmp, -1);
2440 
2441     // Reload values from the stack so they are where the stub
2442     // expects them.
2443     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2444     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2445     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2446     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2447     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2448 
2449     __ subl(length, tmp);
2450     __ addl(src_pos, tmp);
2451     __ addl(dst_pos, tmp);
2452     __ jmp(*stub->entry());
2453 
2454     __ bind(*stub->continuation());
2455     return;
2456   }
2457 
2458   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2459 
2460   int elem_size = type2aelembytes(basic_type);
2461   Address::ScaleFactor scale;
2462 
2463   switch (elem_size) {
2464     case 1 :
2465       scale = Address::times_1;
2466       break;
2467     case 2 :
2468       scale = Address::times_2;
2469       break;
2470     case 4 :
2471       scale = Address::times_4;
2472       break;
2473     case 8 :
2474       scale = Address::times_8;
2475       break;
2476     default:
2477       scale = Address::no_scale;
2478       ShouldNotReachHere();
2479   }
2480 
2481   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2482   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2483 
2484   // length and pos's are all sign extended at this point on 64bit
2485 
2486   // test for null
2487   if (flags & LIR_OpArrayCopy::src_null_check) {
2488     __ testptr(src, src);
2489     __ jcc(Assembler::zero, *stub->entry());
2490   }
2491   if (flags & LIR_OpArrayCopy::dst_null_check) {
2492     __ testptr(dst, dst);
2493     __ jcc(Assembler::zero, *stub->entry());
2494   }
2495 
2496   // If the compiler was not able to prove that exact type of the source or the destination
2497   // of the arraycopy is an array type, check at runtime if the source or the destination is
2498   // an instance type.
2499   if (flags & LIR_OpArrayCopy::type_check) {
2500     if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2501       __ load_klass(tmp, dst, tmp_load_klass);
2502       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2503       __ jcc(Assembler::greaterEqual, *stub->entry());
2504     }
2505 
2506     if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2507       __ load_klass(tmp, src, tmp_load_klass);
2508       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
2509       __ jcc(Assembler::greaterEqual, *stub->entry());
2510     }
2511   }
2512 
2513   // check if negative
2514   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2515     __ testl(src_pos, src_pos);
2516     __ jcc(Assembler::less, *stub->entry());
2517   }
2518   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2519     __ testl(dst_pos, dst_pos);
2520     __ jcc(Assembler::less, *stub->entry());
2521   }
2522 
2523   if (flags & LIR_OpArrayCopy::src_range_check) {
2524     __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
2525     __ cmpl(tmp, src_length_addr);
2526     __ jcc(Assembler::above, *stub->entry());
2527   }
2528   if (flags & LIR_OpArrayCopy::dst_range_check) {
2529     __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
2530     __ cmpl(tmp, dst_length_addr);
2531     __ jcc(Assembler::above, *stub->entry());
2532   }
2533 
2534   if (flags & LIR_OpArrayCopy::length_positive_check) {
2535     __ testl(length, length);
2536     __ jcc(Assembler::less, *stub->entry());
2537   }
2538 
2539   __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
2540   __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
2541 
2542   if (flags & LIR_OpArrayCopy::type_check) {
2543     // We don't know the array types are compatible
2544     if (basic_type != T_OBJECT) {
2545       // Simple test for basic type arrays
2546       __ cmp_klasses_from_objects(src, dst, tmp, tmp2);
2547       __ jcc(Assembler::notEqual, *stub->entry());
2548     } else {
2549       // For object arrays, if src is a sub class of dst then we can
2550       // safely do the copy.
2551       Label cont, slow;
2552 
2553       __ push(src);
2554       __ push(dst);
2555 
2556       __ load_klass(src, src, tmp_load_klass);
2557       __ load_klass(dst, dst, tmp_load_klass);
2558 
2559       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2560 
2561       __ push(src);
2562       __ push(dst);
2563       __ call(RuntimeAddress(Runtime1::entry_for(C1StubId::slow_subtype_check_id)));
2564       __ pop(dst);
2565       __ pop(src);
2566 
2567       __ testl(src, src);
2568       __ jcc(Assembler::notEqual, cont);
2569 
2570       __ bind(slow);
2571       __ pop(dst);
2572       __ pop(src);
2573 
2574       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2575       if (copyfunc_addr != nullptr) { // use stub if available
2576         // src is not a sub class of dst so we have to do a
2577         // per-element check.
2578 
2579         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2580         if ((flags & mask) != mask) {
2581           // Check that at least both of them object arrays.
2582           assert(flags & mask, "one of the two should be known to be an object array");
2583 
2584           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2585             __ load_klass(tmp, src, tmp_load_klass);
2586           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2587             __ load_klass(tmp, dst, tmp_load_klass);
2588           }
2589           int lh_offset = in_bytes(Klass::layout_helper_offset());
2590           Address klass_lh_addr(tmp, lh_offset);
2591           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2592           __ cmpl(klass_lh_addr, objArray_lh);
2593           __ jcc(Assembler::notEqual, *stub->entry());
2594         }
2595 
2596        // Spill because stubs can use any register they like and it's
2597        // easier to restore just those that we care about.
2598        store_parameter(dst, 0);
2599        store_parameter(dst_pos, 1);
2600        store_parameter(length, 2);
2601        store_parameter(src_pos, 3);
2602        store_parameter(src, 4);
2603 
2604         __ movl2ptr(length, length); //higher 32bits must be null
2605 
2606         __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2607         assert_different_registers(c_rarg0, dst, dst_pos, length);
2608         __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2609         assert_different_registers(c_rarg1, dst, length);
2610 
2611         __ mov(c_rarg2, length);
2612         assert_different_registers(c_rarg2, dst);
2613 
2614 #ifdef _WIN64
2615         // Allocate abi space for args but be sure to keep stack aligned
2616         __ subptr(rsp, 6*wordSize);
2617         __ load_klass(c_rarg3, dst, tmp_load_klass);
2618         __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
2619         store_parameter(c_rarg3, 4);
2620         __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
2621         __ call(RuntimeAddress(copyfunc_addr));
2622         __ addptr(rsp, 6*wordSize);
2623 #else
2624         __ load_klass(c_rarg4, dst, tmp_load_klass);
2625         __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2626         __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2627         __ call(RuntimeAddress(copyfunc_addr));
2628 #endif
2629 
2630 #ifndef PRODUCT
2631         if (PrintC1Statistics) {
2632           Label failed;
2633           __ testl(rax, rax);
2634           __ jcc(Assembler::notZero, failed);
2635           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
2636           __ bind(failed);
2637         }
2638 #endif
2639 
2640         __ testl(rax, rax);
2641         __ jcc(Assembler::zero, *stub->continuation());
2642 
2643 #ifndef PRODUCT
2644         if (PrintC1Statistics) {
2645           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
2646         }
2647 #endif
2648 
2649         __ mov(tmp, rax);
2650 
2651         __ xorl(tmp, -1);
2652 
2653         // Restore previously spilled arguments
2654         __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2655         __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2656         __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2657         __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2658         __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2659 
2660 
2661         __ subl(length, tmp);
2662         __ addl(src_pos, tmp);
2663         __ addl(dst_pos, tmp);
2664       }
2665 
2666       __ jmp(*stub->entry());
2667 
2668       __ bind(cont);
2669       __ pop(dst);
2670       __ pop(src);
2671     }
2672   }
2673 
2674 #ifdef ASSERT
2675   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2676     // Sanity check the known type with the incoming class.  For the
2677     // primitive case the types must match exactly with src.klass and
2678     // dst.klass each exactly matching the default type.  For the
2679     // object array case, if no type check is needed then either the
2680     // dst type is exactly the expected type and the src type is a
2681     // subtype which we can't check or src is the same array as dst
2682     // but not necessarily exactly of type default_type.
2683     Label known_ok, halt;
2684     __ mov_metadata(tmp, default_type->constant_encoding());
2685     if (UseCompressedClassPointers) {
2686       __ encode_klass_not_null(tmp, rscratch1);
2687     }
2688 
2689     if (basic_type != T_OBJECT) {
2690       __ cmp_klass(tmp, dst, tmp2);
2691       __ jcc(Assembler::notEqual, halt);
2692       __ cmp_klass(tmp, src, tmp2);
2693       __ jcc(Assembler::equal, known_ok);
2694     } else {
2695       __ cmp_klass(tmp, dst, tmp2);
2696       __ jcc(Assembler::equal, known_ok);
2697       __ cmpptr(src, dst);
2698       __ jcc(Assembler::equal, known_ok);
2699     }
2700     __ bind(halt);
2701     __ stop("incorrect type information in arraycopy");
2702     __ bind(known_ok);
2703   }
2704 #endif
2705 
2706 #ifndef PRODUCT
2707   if (PrintC1Statistics) {
2708     __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)), rscratch1);
2709   }
2710 #endif
2711 
2712   assert_different_registers(c_rarg0, dst, dst_pos, length);
2713   __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2714   assert_different_registers(c_rarg1, length);
2715   __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
2716   __ mov(c_rarg2, length);
2717 
2718   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2719   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2720   const char *name;
2721   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2722   __ call_VM_leaf(entry, 0);
2723 
2724   if (stub != nullptr) {
2725     __ bind(*stub->continuation());
2726   }
2727 }
2728 
2729 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2730   assert(op->crc()->is_single_cpu(),  "crc must be register");
2731   assert(op->val()->is_single_cpu(),  "byte value must be register");
2732   assert(op->result_opr()->is_single_cpu(), "result must be register");
2733   Register crc = op->crc()->as_register();
2734   Register val = op->val()->as_register();
2735   Register res = op->result_opr()->as_register();
2736 
2737   assert_different_registers(val, crc, res);
2738 
2739   __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2740   __ notl(crc); // ~crc
2741   __ update_byte_crc32(crc, val, res);
2742   __ notl(crc); // ~crc
2743   __ mov(res, crc);
2744 }
2745 
2746 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2747   Register obj = op->obj_opr()->as_register();  // may not be an oop
2748   Register hdr = op->hdr_opr()->as_register();
2749   Register lock = op->lock_opr()->as_register();
2750   if (LockingMode == LM_MONITOR) {
2751     if (op->info() != nullptr) {
2752       add_debug_info_for_null_check_here(op->info());
2753       __ null_check(obj);
2754     }
2755     __ jmp(*op->stub()->entry());
2756   } else if (op->code() == lir_lock) {
2757     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2758     Register tmp = LockingMode == LM_LIGHTWEIGHT ? op->scratch_opr()->as_register() : noreg;
2759     // add debug info for NullPointerException only if one is possible
2760     int null_check_offset = __ lock_object(hdr, obj, lock, tmp, *op->stub()->entry());
2761     if (op->info() != nullptr) {
2762       add_debug_info_for_null_check(null_check_offset, op->info());
2763     }
2764     // done
2765   } else if (op->code() == lir_unlock) {
2766     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2767     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2768   } else {
2769     Unimplemented();
2770   }
2771   __ bind(*op->stub()->continuation());
2772 }
2773 
2774 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2775   Register obj = op->obj()->as_pointer_register();
2776   Register result = op->result_opr()->as_pointer_register();
2777 
2778   CodeEmitInfo* info = op->info();
2779   if (info != nullptr) {
2780     add_debug_info_for_null_check_here(info);
2781   }
2782 
2783   __ load_klass(result, obj, rscratch1);
2784 }
2785 
2786 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2787   ciMethod* method = op->profiled_method();
2788   int bci          = op->profiled_bci();
2789   ciMethod* callee = op->profiled_callee();
2790   Register tmp_load_klass = rscratch1;
2791 
2792   // Update counter for all call types
2793   ciMethodData* md = method->method_data_or_null();
2794   assert(md != nullptr, "Sanity");
2795   ciProfileData* data = md->bci_to_data(bci);
2796   assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2797   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2798   Register mdo  = op->mdo()->as_register();
2799   __ mov_metadata(mdo, md->constant_encoding());
2800   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2801   // Perform additional virtual call profiling for invokevirtual and
2802   // invokeinterface bytecodes
2803   if (op->should_profile_receiver_type()) {
2804     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2805     Register recv = op->recv()->as_register();
2806     assert_different_registers(mdo, recv);
2807     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2808     ciKlass* known_klass = op->known_holder();
2809     if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2810       // We know the type that will be seen at this call site; we can
2811       // statically update the MethodData* rather than needing to do
2812       // dynamic tests on the receiver type
2813 
2814       // NOTE: we should probably put a lock around this search to
2815       // avoid collisions by concurrent compilations
2816       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2817       uint i;
2818       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2819         ciKlass* receiver = vc_data->receiver(i);
2820         if (known_klass->equals(receiver)) {
2821           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2822           __ addptr(data_addr, DataLayout::counter_increment);
2823           return;
2824         }
2825       }
2826 
2827       // Receiver type not found in profile data; select an empty slot
2828 
2829       // Note that this is less efficient than it should be because it
2830       // always does a write to the receiver part of the
2831       // VirtualCallData rather than just the first time
2832       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2833         ciKlass* receiver = vc_data->receiver(i);
2834         if (receiver == nullptr) {
2835           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2836           __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
2837           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2838           __ addptr(data_addr, DataLayout::counter_increment);
2839           return;
2840         }
2841       }
2842     } else {
2843       __ load_klass(recv, recv, tmp_load_klass);
2844       Label update_done;
2845       type_profile_helper(mdo, md, data, recv, &update_done);
2846       // Receiver did not match any saved receiver and there is no empty row for it.
2847       // Increment total counter to indicate polymorphic case.
2848       __ addptr(counter_addr, DataLayout::counter_increment);
2849 
2850       __ bind(update_done);
2851     }
2852   } else {
2853     // Static call
2854     __ addptr(counter_addr, DataLayout::counter_increment);
2855   }
2856 }
2857 
2858 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2859   Register obj = op->obj()->as_register();
2860   Register tmp = op->tmp()->as_pointer_register();
2861   Register tmp_load_klass = rscratch1;
2862   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2863   ciKlass* exact_klass = op->exact_klass();
2864   intptr_t current_klass = op->current_klass();
2865   bool not_null = op->not_null();
2866   bool no_conflict = op->no_conflict();
2867 
2868   Label update, next, none;
2869 
2870   bool do_null = !not_null;
2871   bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2872   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2873 
2874   assert(do_null || do_update, "why are we here?");
2875   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2876 
2877   __ verify_oop(obj);
2878 
2879 #ifdef ASSERT
2880   if (obj == tmp) {
2881     assert_different_registers(obj, rscratch1, mdo_addr.base(), mdo_addr.index());
2882   } else {
2883     assert_different_registers(obj, tmp, rscratch1, mdo_addr.base(), mdo_addr.index());
2884   }
2885 #endif
2886   if (do_null) {
2887     __ testptr(obj, obj);
2888     __ jccb(Assembler::notZero, update);
2889     if (!TypeEntries::was_null_seen(current_klass)) {
2890       __ testptr(mdo_addr, TypeEntries::null_seen);
2891 #ifndef ASSERT
2892       __ jccb(Assembler::notZero, next); // already set
2893 #else
2894       __ jcc(Assembler::notZero, next); // already set
2895 #endif
2896       // atomic update to prevent overwriting Klass* with 0
2897       __ lock();
2898       __ orptr(mdo_addr, TypeEntries::null_seen);
2899     }
2900     if (do_update) {
2901 #ifndef ASSERT
2902       __ jmpb(next);
2903     }
2904 #else
2905       __ jmp(next);
2906     }
2907   } else {
2908     __ testptr(obj, obj);
2909     __ jcc(Assembler::notZero, update);
2910     __ stop("unexpected null obj");
2911 #endif
2912   }
2913 
2914   __ bind(update);
2915 
2916   if (do_update) {
2917 #ifdef ASSERT
2918     if (exact_klass != nullptr) {
2919       Label ok;
2920       __ load_klass(tmp, obj, tmp_load_klass);
2921       __ push(tmp);
2922       __ mov_metadata(tmp, exact_klass->constant_encoding());
2923       __ cmpptr(tmp, Address(rsp, 0));
2924       __ jcc(Assembler::equal, ok);
2925       __ stop("exact klass and actual klass differ");
2926       __ bind(ok);
2927       __ pop(tmp);
2928     }
2929 #endif
2930     if (!no_conflict) {
2931       if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
2932         if (exact_klass != nullptr) {
2933           __ mov_metadata(tmp, exact_klass->constant_encoding());
2934         } else {
2935           __ load_klass(tmp, obj, tmp_load_klass);
2936         }
2937         __ mov(rscratch1, tmp); // save original value before XOR
2938         __ xorptr(tmp, mdo_addr);
2939         __ testptr(tmp, TypeEntries::type_klass_mask);
2940         // klass seen before, nothing to do. The unknown bit may have been
2941         // set already but no need to check.
2942         __ jccb(Assembler::zero, next);
2943 
2944         __ testptr(tmp, TypeEntries::type_unknown);
2945         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2946 
2947         if (TypeEntries::is_type_none(current_klass)) {
2948           __ testptr(mdo_addr, TypeEntries::type_mask);
2949           __ jccb(Assembler::zero, none);
2950           // There is a chance that the checks above (re-reading profiling
2951           // data from memory) fail if another thread has just set the
2952           // profiling to this obj's klass
2953           __ mov(tmp, rscratch1); // get back original value before XOR
2954           __ xorptr(tmp, mdo_addr);
2955           __ testptr(tmp, TypeEntries::type_klass_mask);
2956           __ jccb(Assembler::zero, next);
2957         }
2958       } else {
2959         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2960                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2961 
2962         __ testptr(mdo_addr, TypeEntries::type_unknown);
2963         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
2964       }
2965 
2966       // different than before. Cannot keep accurate profile.
2967       __ orptr(mdo_addr, TypeEntries::type_unknown);
2968 
2969       if (TypeEntries::is_type_none(current_klass)) {
2970         __ jmpb(next);
2971 
2972         __ bind(none);
2973         // first time here. Set profile type.
2974         __ movptr(mdo_addr, tmp);
2975 #ifdef ASSERT
2976         __ andptr(tmp, TypeEntries::type_klass_mask);
2977         __ verify_klass_ptr(tmp);
2978 #endif
2979       }
2980     } else {
2981       // There's a single possible klass at this profile point
2982       assert(exact_klass != nullptr, "should be");
2983       if (TypeEntries::is_type_none(current_klass)) {
2984         __ mov_metadata(tmp, exact_klass->constant_encoding());
2985         __ xorptr(tmp, mdo_addr);
2986         __ testptr(tmp, TypeEntries::type_klass_mask);
2987 #ifdef ASSERT
2988         __ jcc(Assembler::zero, next);
2989 
2990         {
2991           Label ok;
2992           __ push(tmp);
2993           __ testptr(mdo_addr, TypeEntries::type_mask);
2994           __ jcc(Assembler::zero, ok);
2995           // may have been set by another thread
2996           __ mov_metadata(tmp, exact_klass->constant_encoding());
2997           __ xorptr(tmp, mdo_addr);
2998           __ testptr(tmp, TypeEntries::type_mask);
2999           __ jcc(Assembler::zero, ok);
3000 
3001           __ stop("unexpected profiling mismatch");
3002           __ bind(ok);
3003           __ pop(tmp);
3004         }
3005 #else
3006         __ jccb(Assembler::zero, next);
3007 #endif
3008         // first time here. Set profile type.
3009         __ movptr(mdo_addr, tmp);
3010 #ifdef ASSERT
3011         __ andptr(tmp, TypeEntries::type_klass_mask);
3012         __ verify_klass_ptr(tmp);
3013 #endif
3014       } else {
3015         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3016                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3017 
3018         __ testptr(mdo_addr, TypeEntries::type_unknown);
3019         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3020 
3021         __ orptr(mdo_addr, TypeEntries::type_unknown);
3022       }
3023     }
3024   }
3025   __ bind(next);
3026 }
3027 
3028 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3029   Unimplemented();
3030 }
3031 
3032 
3033 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3034   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3035 }
3036 
3037 
3038 void LIR_Assembler::align_backward_branch_target() {
3039   __ align(BytesPerWord);
3040 }
3041 
3042 
3043 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3044   if (left->is_single_cpu()) {
3045     __ negl(left->as_register());
3046     move_regs(left->as_register(), dest->as_register());
3047 
3048   } else if (left->is_double_cpu()) {
3049     Register lo = left->as_register_lo();
3050     Register dst = dest->as_register_lo();
3051     __ movptr(dst, lo);
3052     __ negptr(dst);
3053 
3054   } else if (dest->is_single_xmm()) {
3055     assert(!tmp->is_valid(), "do not need temporary");
3056     if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3057       __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3058     }
3059     __ xorps(dest->as_xmm_float_reg(),
3060              ExternalAddress((address)float_signflip_pool),
3061              rscratch1);
3062   } else if (dest->is_double_xmm()) {
3063     assert(!tmp->is_valid(), "do not need temporary");
3064     if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3065       __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3066     }
3067     __ xorpd(dest->as_xmm_double_reg(),
3068              ExternalAddress((address)double_signflip_pool),
3069              rscratch1);
3070   } else {
3071     ShouldNotReachHere();
3072   }
3073 }
3074 
3075 
3076 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3077   assert(src->is_address(), "must be an address");
3078   assert(dest->is_register(), "must be a register");
3079 
3080   PatchingStub* patch = nullptr;
3081   if (patch_code != lir_patch_none) {
3082     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3083   }
3084 
3085   Register reg = dest->as_pointer_register();
3086   LIR_Address* addr = src->as_address_ptr();
3087   __ lea(reg, as_Address(addr));
3088 
3089   if (patch != nullptr) {
3090     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
3091   }
3092 }
3093 
3094 
3095 
3096 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3097   assert(!tmp->is_valid(), "don't need temporary");
3098   __ call(RuntimeAddress(dest));
3099   if (info != nullptr) {
3100     add_call_info_here(info);
3101   }
3102   __ post_call_nop();
3103 }
3104 
3105 
3106 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3107   assert(type == T_LONG, "only for volatile long fields");
3108 
3109   if (info != nullptr) {
3110     add_debug_info_for_null_check_here(info);
3111   }
3112 
3113   if (src->is_double_xmm()) {
3114     if (dest->is_double_cpu()) {
3115       __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
3116     } else if (dest->is_double_stack()) {
3117       __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
3118     } else if (dest->is_address()) {
3119       __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
3120     } else {
3121       ShouldNotReachHere();
3122     }
3123 
3124   } else if (dest->is_double_xmm()) {
3125     if (src->is_double_stack()) {
3126       __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
3127     } else if (src->is_address()) {
3128       __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
3129     } else {
3130       ShouldNotReachHere();
3131     }
3132 
3133   } else {
3134     ShouldNotReachHere();
3135   }
3136 }
3137 
3138 #ifdef ASSERT
3139 // emit run-time assertion
3140 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3141   assert(op->code() == lir_assert, "must be");
3142 
3143   if (op->in_opr1()->is_valid()) {
3144     assert(op->in_opr2()->is_valid(), "both operands must be valid");
3145     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3146   } else {
3147     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3148     assert(op->condition() == lir_cond_always, "no other conditions allowed");
3149   }
3150 
3151   Label ok;
3152   if (op->condition() != lir_cond_always) {
3153     Assembler::Condition acond = Assembler::zero;
3154     switch (op->condition()) {
3155       case lir_cond_equal:        acond = Assembler::equal;       break;
3156       case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
3157       case lir_cond_less:         acond = Assembler::less;        break;
3158       case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
3159       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
3160       case lir_cond_greater:      acond = Assembler::greater;     break;
3161       case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
3162       case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
3163       default:                    ShouldNotReachHere();
3164     }
3165     __ jcc(acond, ok);
3166   }
3167   if (op->halt()) {
3168     const char* str = __ code_string(op->msg());
3169     __ stop(str);
3170   } else {
3171     breakpoint();
3172   }
3173   __ bind(ok);
3174 }
3175 #endif
3176 
3177 void LIR_Assembler::membar() {
3178   // QQQ sparc TSO uses this,
3179   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
3180 }
3181 
3182 void LIR_Assembler::membar_acquire() {
3183   // No x86 machines currently require load fences
3184 }
3185 
3186 void LIR_Assembler::membar_release() {
3187   // No x86 machines currently require store fences
3188 }
3189 
3190 void LIR_Assembler::membar_loadload() {
3191   // no-op
3192   //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3193 }
3194 
3195 void LIR_Assembler::membar_storestore() {
3196   // no-op
3197   //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3198 }
3199 
3200 void LIR_Assembler::membar_loadstore() {
3201   // no-op
3202   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3203 }
3204 
3205 void LIR_Assembler::membar_storeload() {
3206   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3207 }
3208 
3209 void LIR_Assembler::on_spin_wait() {
3210   __ pause ();
3211 }
3212 
3213 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3214   assert(result_reg->is_register(), "check");
3215   __ mov(result_reg->as_register(), r15_thread);
3216 }
3217 
3218 
3219 void LIR_Assembler::peephole(LIR_List*) {
3220   // do nothing for now
3221 }
3222 
3223 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3224   assert(data == dest, "xchg/xadd uses only 2 operands");
3225 
3226   if (data->type() == T_INT) {
3227     if (code == lir_xadd) {
3228       __ lock();
3229       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3230     } else {
3231       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3232     }
3233   } else if (data->is_oop()) {
3234     assert (code == lir_xchg, "xadd for oops");
3235     Register obj = data->as_register();
3236     if (UseCompressedOops) {
3237       __ encode_heap_oop(obj);
3238       __ xchgl(obj, as_Address(src->as_address_ptr()));
3239       __ decode_heap_oop(obj);
3240     } else {
3241       __ xchgptr(obj, as_Address(src->as_address_ptr()));
3242     }
3243   } else if (data->type() == T_LONG) {
3244     assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
3245     if (code == lir_xadd) {
3246       __ lock();
3247       __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
3248     } else {
3249       __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
3250     }
3251   } else {
3252     ShouldNotReachHere();
3253   }
3254 }
3255 
3256 #undef __