1 /*
   2  * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "c1/c1_CodeStubs.hpp"
  29 #include "c1/c1_Compilation.hpp"
  30 #include "c1/c1_LIRAssembler.hpp"
  31 #include "c1/c1_MacroAssembler.hpp"
  32 #include "c1/c1_Runtime1.hpp"
  33 #include "c1/c1_ValueStack.hpp"
  34 #include "ci/ciArrayKlass.hpp"
  35 #include "ci/ciInlineKlass.hpp"
  36 #include "ci/ciInstance.hpp"
  37 #include "compiler/oopMap.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "gc/shared/gc_globals.hpp"
  40 #include "nativeInst_x86.hpp"
  41 #include "oops/oop.inline.hpp"
  42 #include "oops/objArrayKlass.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/safepointMechanism.hpp"
  45 #include "runtime/sharedRuntime.hpp"
  46 #include "runtime/stubRoutines.hpp"
  47 #include "utilities/powerOfTwo.hpp"
  48 #include "vmreg_x86.inline.hpp"
  49 
  50 
  51 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  52 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  53 // fast versions of NegF/NegD and AbsF/AbsD.
  54 
  55 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  56 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  57   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  58   // of 128-bits operands for SSE instructions.
  59   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  60   // Store the value to a 128-bits operand.
  61   operand[0] = lo;
  62   operand[1] = hi;
  63   return operand;
  64 }
  65 
  66 // Buffer for 128-bits masks used by SSE instructions.
  67 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
  68 
  69 // Static initialization during VM startup.
  70 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2],         CONST64(0x7FFFFFFF7FFFFFFF),         CONST64(0x7FFFFFFF7FFFFFFF));
  71 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2],         CONST64(0x7FFFFFFFFFFFFFFF),         CONST64(0x7FFFFFFFFFFFFFFF));
  72 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
  73 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
  74 
  75 
  76 NEEDS_CLEANUP // remove this definitions ?
  77 const Register IC_Klass    = rax;   // where the IC klass is cached
  78 const Register SYNC_header = rax;   // synchronization header
  79 const Register SHIFT_count = rcx;   // where count for shift operations must be
  80 
  81 #define __ _masm->
  82 
  83 
  84 static void select_different_registers(Register preserve,
  85                                        Register extra,
  86                                        Register &tmp1,
  87                                        Register &tmp2) {
  88   if (tmp1 == preserve) {
  89     assert_different_registers(tmp1, tmp2, extra);
  90     tmp1 = extra;
  91   } else if (tmp2 == preserve) {
  92     assert_different_registers(tmp1, tmp2, extra);
  93     tmp2 = extra;
  94   }
  95   assert_different_registers(preserve, tmp1, tmp2);
  96 }
  97 
  98 
  99 
 100 static void select_different_registers(Register preserve,
 101                                        Register extra,
 102                                        Register &tmp1,
 103                                        Register &tmp2,
 104                                        Register &tmp3) {
 105   if (tmp1 == preserve) {
 106     assert_different_registers(tmp1, tmp2, tmp3, extra);
 107     tmp1 = extra;
 108   } else if (tmp2 == preserve) {
 109     assert_different_registers(tmp1, tmp2, tmp3, extra);
 110     tmp2 = extra;
 111   } else if (tmp3 == preserve) {
 112     assert_different_registers(tmp1, tmp2, tmp3, extra);
 113     tmp3 = extra;
 114   }
 115   assert_different_registers(preserve, tmp1, tmp2, tmp3);
 116 }
 117 
 118 
 119 
 120 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
 121   if (opr->is_constant()) {
 122     LIR_Const* constant = opr->as_constant_ptr();
 123     switch (constant->type()) {
 124       case T_INT: {
 125         return true;
 126       }
 127 
 128       default:
 129         return false;
 130     }
 131   }
 132   return false;
 133 }
 134 
 135 
 136 LIR_Opr LIR_Assembler::receiverOpr() {
 137   return FrameMap::receiver_opr;
 138 }
 139 
 140 LIR_Opr LIR_Assembler::osrBufferPointer() {
 141   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 142 }
 143 
 144 //--------------fpu register translations-----------------------
 145 
 146 
 147 address LIR_Assembler::float_constant(float f) {
 148   address const_addr = __ float_constant(f);
 149   if (const_addr == NULL) {
 150     bailout("const section overflow");
 151     return __ code()->consts()->start();
 152   } else {
 153     return const_addr;
 154   }
 155 }
 156 
 157 
 158 address LIR_Assembler::double_constant(double d) {
 159   address const_addr = __ double_constant(d);
 160   if (const_addr == NULL) {
 161     bailout("const section overflow");
 162     return __ code()->consts()->start();
 163   } else {
 164     return const_addr;
 165   }
 166 }
 167 
 168 #ifndef _LP64
 169 void LIR_Assembler::fpop() {
 170   __ fpop();
 171 }
 172 
 173 void LIR_Assembler::fxch(int i) {
 174   __ fxch(i);
 175 }
 176 
 177 void LIR_Assembler::fld(int i) {
 178   __ fld_s(i);
 179 }
 180 
 181 void LIR_Assembler::ffree(int i) {
 182   __ ffree(i);
 183 }
 184 #endif // !_LP64
 185 
 186 void LIR_Assembler::breakpoint() {
 187   __ int3();
 188 }
 189 
 190 void LIR_Assembler::push(LIR_Opr opr) {
 191   if (opr->is_single_cpu()) {
 192     __ push_reg(opr->as_register());
 193   } else if (opr->is_double_cpu()) {
 194     NOT_LP64(__ push_reg(opr->as_register_hi()));
 195     __ push_reg(opr->as_register_lo());
 196   } else if (opr->is_stack()) {
 197     __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
 198   } else if (opr->is_constant()) {
 199     LIR_Const* const_opr = opr->as_constant_ptr();
 200     if (const_opr->type() == T_OBJECT || const_opr->type() == T_PRIMITIVE_OBJECT) {
 201       __ push_oop(const_opr->as_jobject(), rscratch1);
 202     } else if (const_opr->type() == T_INT) {
 203       __ push_jint(const_opr->as_jint());
 204     } else {
 205       ShouldNotReachHere();
 206     }
 207 
 208   } else {
 209     ShouldNotReachHere();
 210   }
 211 }
 212 
 213 void LIR_Assembler::pop(LIR_Opr opr) {
 214   if (opr->is_single_cpu()) {
 215     __ pop_reg(opr->as_register());
 216   } else {
 217     ShouldNotReachHere();
 218   }
 219 }
 220 
 221 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
 222   return addr->base()->is_illegal() && addr->index()->is_illegal();
 223 }
 224 
 225 //-------------------------------------------
 226 
 227 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 228   return as_Address(addr, rscratch1);
 229 }
 230 
 231 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 232   if (addr->base()->is_illegal()) {
 233     assert(addr->index()->is_illegal(), "must be illegal too");
 234     AddressLiteral laddr((address)addr->disp(), relocInfo::none);
 235     if (! __ reachable(laddr)) {
 236       __ movptr(tmp, laddr.addr());
 237       Address res(tmp, 0);
 238       return res;
 239     } else {
 240       return __ as_Address(laddr);
 241     }
 242   }
 243 
 244   Register base = addr->base()->as_pointer_register();
 245 
 246   if (addr->index()->is_illegal()) {
 247     return Address( base, addr->disp());
 248   } else if (addr->index()->is_cpu_register()) {
 249     Register index = addr->index()->as_pointer_register();
 250     return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
 251   } else if (addr->index()->is_constant()) {
 252     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 253     assert(Assembler::is_simm32(addr_offset), "must be");
 254 
 255     return Address(base, addr_offset);
 256   } else {
 257     Unimplemented();
 258     return Address();
 259   }
 260 }
 261 
 262 
 263 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 264   Address base = as_Address(addr);
 265   return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
 266 }
 267 
 268 
 269 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 270   return as_Address(addr);
 271 }
 272 
 273 
 274 void LIR_Assembler::osr_entry() {
 275   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 276   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 277   ValueStack* entry_state = osr_entry->state();
 278   int number_of_locks = entry_state->locks_size();
 279 
 280   // we jump here if osr happens with the interpreter
 281   // state set up to continue at the beginning of the
 282   // loop that triggered osr - in particular, we have
 283   // the following registers setup:
 284   //
 285   // rcx: osr buffer
 286   //
 287 
 288   // build frame
 289   ciMethod* m = compilation()->method();
 290   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 291 
 292   // OSR buffer is
 293   //
 294   // locals[nlocals-1..0]
 295   // monitors[0..number_of_locks]
 296   //
 297   // locals is a direct copy of the interpreter frame so in the osr buffer
 298   // so first slot in the local array is the last local from the interpreter
 299   // and last slot is local[0] (receiver) from the interpreter
 300   //
 301   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 302   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 303   // in the interpreter frame (the method lock if a sync method)
 304 
 305   // Initialize monitors in the compiled activation.
 306   //   rcx: pointer to osr buffer
 307   //
 308   // All other registers are dead at this point and the locals will be
 309   // copied into place by code emitted in the IR.
 310 
 311   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 312   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 313     int monitor_offset = BytesPerWord * method()->max_locals() +
 314       (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
 315     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 316     // the OSR buffer using 2 word entries: first the lock and then
 317     // the oop.
 318     for (int i = 0; i < number_of_locks; i++) {
 319       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 320 #ifdef ASSERT
 321       // verify the interpreter's monitor has a non-null object
 322       {
 323         Label L;
 324         __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), NULL_WORD);
 325         __ jcc(Assembler::notZero, L);
 326         __ stop("locked object is NULL");
 327         __ bind(L);
 328       }
 329 #endif
 330       __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
 331       __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
 332       __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 333       __ movptr(frame_map()->address_for_monitor_object(i), rbx);
 334     }
 335   }
 336 }
 337 
 338 
 339 // inline cache check; done before the frame is built.
 340 int LIR_Assembler::check_icache() {
 341   Register receiver = FrameMap::receiver_opr->as_register();
 342   Register ic_klass = IC_Klass;
 343   const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
 344   const bool do_post_padding = VerifyOops || UseCompressedClassPointers;
 345   if (!do_post_padding) {
 346     // insert some nops so that the verified entry point is aligned on CodeEntryAlignment
 347     __ align(CodeEntryAlignment, __ offset() + ic_cmp_size);
 348   }
 349   int offset = __ offset();
 350   __ inline_cache_check(receiver, IC_Klass);
 351   assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct");
 352   if (do_post_padding) {
 353     // force alignment after the cache check.
 354     // It's been verified to be aligned if !VerifyOops
 355     __ align(CodeEntryAlignment);
 356   }
 357   return offset;
 358 }
 359 
 360 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 361   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 362   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 363 
 364   Label L_skip_barrier;
 365   Register klass = rscratch1;
 366   Register thread = LP64_ONLY( r15_thread ) NOT_LP64( noreg );
 367   assert(thread != noreg, "x86_32 not implemented");
 368 
 369   __ mov_metadata(klass, method->holder()->constant_encoding());
 370   __ clinit_barrier(klass, thread, &L_skip_barrier /*L_fast_path*/);
 371 
 372   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 373 
 374   __ bind(L_skip_barrier);
 375 }
 376 
 377 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
 378   jobject o = NULL;
 379   PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
 380   __ movoop(reg, o);
 381   patching_epilog(patch, lir_patch_normal, reg, info);
 382 }
 383 
 384 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 385   Metadata* o = NULL;
 386   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 387   __ mov_metadata(reg, o);
 388   patching_epilog(patch, lir_patch_normal, reg, info);
 389 }
 390 
 391 // This specifies the rsp decrement needed to build the frame
 392 int LIR_Assembler::initial_frame_size_in_bytes() const {
 393   // if rounding, must let FrameMap know!
 394 
 395   // The frame_map records size in slots (32bit word)
 396 
 397   // subtract two words to account for return address and link
 398   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 399 }
 400 
 401 
 402 int LIR_Assembler::emit_exception_handler() {
 403   // generate code for exception handler
 404   address handler_base = __ start_a_stub(exception_handler_size());
 405   if (handler_base == NULL) {
 406     // not enough space left for the handler
 407     bailout("exception handler overflow");
 408     return -1;
 409   }
 410 
 411   int offset = code_offset();
 412 
 413   // the exception oop and pc are in rax, and rdx
 414   // no other registers need to be preserved, so invalidate them
 415   __ invalidate_registers(false, true, true, false, true, true);
 416 
 417   // check that there is really an exception
 418   __ verify_not_null_oop(rax);
 419 
 420   // search an exception handler (rax: exception oop, rdx: throwing pc)
 421   __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 422   __ should_not_reach_here();
 423   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 424   __ end_a_stub();
 425 
 426   return offset;
 427 }
 428 
 429 
 430 // Emit the code to remove the frame from the stack in the exception
 431 // unwind path.
 432 int LIR_Assembler::emit_unwind_handler() {
 433 #ifndef PRODUCT
 434   if (CommentedAssembly) {
 435     _masm->block_comment("Unwind handler");
 436   }
 437 #endif
 438 
 439   int offset = code_offset();
 440 
 441   // Fetch the exception from TLS and clear out exception related thread state
 442   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 443   NOT_LP64(__ get_thread(thread));
 444   __ movptr(rax, Address(thread, JavaThread::exception_oop_offset()));
 445   __ movptr(Address(thread, JavaThread::exception_oop_offset()), NULL_WORD);
 446   __ movptr(Address(thread, JavaThread::exception_pc_offset()), NULL_WORD);
 447 
 448   __ bind(_unwind_handler_entry);
 449   __ verify_not_null_oop(rax);
 450   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 451     __ mov(rbx, rax);  // Preserve the exception (rbx is always callee-saved)
 452   }
 453 
 454   // Perform needed unlocking
 455   MonitorExitStub* stub = NULL;
 456   if (method()->is_synchronized()) {
 457     monitor_address(0, FrameMap::rax_opr);
 458     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
 459     if (UseHeavyMonitors) {
 460       __ jmp(*stub->entry());
 461     } else {
 462       __ unlock_object(rdi, rsi, rax, *stub->entry());
 463     }
 464     __ bind(*stub->continuation());
 465   }
 466 
 467   if (compilation()->env()->dtrace_method_probes()) {
 468 #ifdef _LP64
 469     __ mov(rdi, r15_thread);
 470     __ mov_metadata(rsi, method()->constant_encoding());
 471 #else
 472     __ get_thread(rax);
 473     __ movptr(Address(rsp, 0), rax);
 474     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
 475 #endif
 476     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 477   }
 478 
 479   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 480     __ mov(rax, rbx);  // Restore the exception
 481   }
 482 
 483   // remove the activation and dispatch to the unwind handler
 484   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 485   __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 486 
 487   // Emit the slow path assembly
 488   if (stub != NULL) {
 489     stub->emit_code(this);
 490   }
 491 
 492   return offset;
 493 }
 494 
 495 
 496 int LIR_Assembler::emit_deopt_handler() {
 497   // generate code for exception handler
 498   address handler_base = __ start_a_stub(deopt_handler_size());
 499   if (handler_base == NULL) {
 500     // not enough space left for the handler
 501     bailout("deopt handler overflow");
 502     return -1;
 503   }
 504 
 505   int offset = code_offset();
 506   InternalAddress here(__ pc());
 507 
 508   __ pushptr(here.addr(), rscratch1);
 509   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 510   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 511   __ end_a_stub();
 512 
 513   return offset;
 514 }
 515 
 516 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 517   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 518   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 519     assert(result->fpu() == 0, "result must already be on TOS");
 520   }
 521 
 522   ciMethod* method = compilation()->method();
 523   if (InlineTypeReturnedAsFields && method->return_type()->is_inlinetype()) {
 524     ciInlineKlass* vk = method->return_type()->as_inline_klass();
 525     if (vk->can_be_returned_as_fields()) {
 526 #ifndef _LP64
 527       Unimplemented();
 528 #else
 529       address unpack_handler = vk->unpack_handler();
 530       assert(unpack_handler != NULL, "must be");
 531       __ call(RuntimeAddress(unpack_handler));
 532       // At this point, rax points to the value object (for interpreter or C1 caller).
 533       // The fields of the object are copied into registers (for C2 caller).
 534 #endif
 535     }
 536   }
 537 
 538   // Pop the stack before the safepoint code
 539   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 540 
 541   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 542     __ reserved_stack_check();
 543   }
 544 
 545   // Note: we do not need to round double result; float result has the right precision
 546   // the poll sets the condition code, but no data registers
 547 
 548 #ifdef _LP64
 549   const Register thread = r15_thread;
 550 #else
 551   const Register thread = rbx;
 552   __ get_thread(thread);
 553 #endif
 554   code_stub->set_safepoint_offset(__ offset());
 555   __ relocate(relocInfo::poll_return_type);
 556   __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
 557   __ ret(0);
 558 }
 559 
 560 
 561 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
 562   return (__ store_inline_type_fields_to_buf(vk, false));
 563 }
 564 
 565 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 566   guarantee(info != NULL, "Shouldn't be NULL");
 567   int offset = __ offset();
 568 #ifdef _LP64
 569   const Register poll_addr = rscratch1;
 570   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 571 #else
 572   assert(tmp->is_cpu_register(), "needed");
 573   const Register poll_addr = tmp->as_register();
 574   __ get_thread(poll_addr);
 575   __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
 576 #endif
 577   add_debug_info_for_branch(info);
 578   __ relocate(relocInfo::poll_type);
 579   address pre_pc = __ pc();
 580   __ testl(rax, Address(poll_addr, 0));
 581   address post_pc = __ pc();
 582   guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
 583   return offset;
 584 }
 585 
 586 
 587 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 588   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 589 }
 590 
 591 void LIR_Assembler::swap_reg(Register a, Register b) {
 592   __ xchgptr(a, b);
 593 }
 594 
 595 
 596 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 597   assert(src->is_constant(), "should not call otherwise");
 598   assert(dest->is_register(), "should not call otherwise");
 599   LIR_Const* c = src->as_constant_ptr();
 600 
 601   switch (c->type()) {
 602     case T_INT: {
 603       assert(patch_code == lir_patch_none, "no patching handled here");
 604       __ movl(dest->as_register(), c->as_jint());
 605       break;
 606     }
 607 
 608     case T_ADDRESS: {
 609       assert(patch_code == lir_patch_none, "no patching handled here");
 610       __ movptr(dest->as_register(), c->as_jint());
 611       break;
 612     }
 613 
 614     case T_LONG: {
 615       assert(patch_code == lir_patch_none, "no patching handled here");
 616 #ifdef _LP64
 617       __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
 618 #else
 619       __ movptr(dest->as_register_lo(), c->as_jint_lo());
 620       __ movptr(dest->as_register_hi(), c->as_jint_hi());
 621 #endif // _LP64
 622       break;
 623     }
 624 
 625     case T_PRIMITIVE_OBJECT: // Fall through
 626     case T_OBJECT: {
 627       if (patch_code != lir_patch_none) {
 628         jobject2reg_with_patching(dest->as_register(), info);
 629       } else {
 630         __ movoop(dest->as_register(), c->as_jobject());
 631       }
 632       break;
 633     }
 634 
 635     case T_METADATA: {
 636       if (patch_code != lir_patch_none) {
 637         klass2reg_with_patching(dest->as_register(), info);
 638       } else {
 639         __ mov_metadata(dest->as_register(), c->as_metadata());
 640       }
 641       break;
 642     }
 643 
 644     case T_FLOAT: {
 645       if (dest->is_single_xmm()) {
 646         if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_float()) {
 647           __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
 648         } else {
 649           __ movflt(dest->as_xmm_float_reg(),
 650                    InternalAddress(float_constant(c->as_jfloat())));
 651         }
 652       } else {
 653 #ifndef _LP64
 654         assert(dest->is_single_fpu(), "must be");
 655         assert(dest->fpu_regnr() == 0, "dest must be TOS");
 656         if (c->is_zero_float()) {
 657           __ fldz();
 658         } else if (c->is_one_float()) {
 659           __ fld1();
 660         } else {
 661           __ fld_s (InternalAddress(float_constant(c->as_jfloat())));
 662         }
 663 #else
 664         ShouldNotReachHere();
 665 #endif // !_LP64
 666       }
 667       break;
 668     }
 669 
 670     case T_DOUBLE: {
 671       if (dest->is_double_xmm()) {
 672         if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_double()) {
 673           __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
 674         } else {
 675           __ movdbl(dest->as_xmm_double_reg(),
 676                     InternalAddress(double_constant(c->as_jdouble())));
 677         }
 678       } else {
 679 #ifndef _LP64
 680         assert(dest->is_double_fpu(), "must be");
 681         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
 682         if (c->is_zero_double()) {
 683           __ fldz();
 684         } else if (c->is_one_double()) {
 685           __ fld1();
 686         } else {
 687           __ fld_d (InternalAddress(double_constant(c->as_jdouble())));
 688         }
 689 #else
 690         ShouldNotReachHere();
 691 #endif // !_LP64
 692       }
 693       break;
 694     }
 695 
 696     default:
 697       ShouldNotReachHere();
 698   }
 699 }
 700 
 701 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 702   assert(src->is_constant(), "should not call otherwise");
 703   assert(dest->is_stack(), "should not call otherwise");
 704   LIR_Const* c = src->as_constant_ptr();
 705 
 706   switch (c->type()) {
 707     case T_INT:  // fall through
 708     case T_FLOAT:
 709       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 710       break;
 711 
 712     case T_ADDRESS:
 713       __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 714       break;
 715 
 716     case T_PRIMITIVE_OBJECT: // Fall through
 717     case T_OBJECT:
 718       __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject(), rscratch1);
 719       break;
 720 
 721     case T_LONG:  // fall through
 722     case T_DOUBLE:
 723 #ifdef _LP64
 724       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 725                                               lo_word_offset_in_bytes),
 726                 (intptr_t)c->as_jlong_bits(),
 727                 rscratch1);
 728 #else
 729       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 730                                               lo_word_offset_in_bytes), c->as_jint_lo_bits());
 731       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 732                                               hi_word_offset_in_bytes), c->as_jint_hi_bits());
 733 #endif // _LP64
 734       break;
 735 
 736     default:
 737       ShouldNotReachHere();
 738   }
 739 }
 740 
 741 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 742   assert(src->is_constant(), "should not call otherwise");
 743   assert(dest->is_address(), "should not call otherwise");
 744   LIR_Const* c = src->as_constant_ptr();
 745   LIR_Address* addr = dest->as_address_ptr();
 746 
 747   int null_check_here = code_offset();
 748   switch (type) {
 749     case T_INT:    // fall through
 750     case T_FLOAT:
 751       __ movl(as_Address(addr), c->as_jint_bits());
 752       break;
 753 
 754     case T_ADDRESS:
 755       __ movptr(as_Address(addr), c->as_jint_bits());
 756       break;
 757 
 758     case T_PRIMITIVE_OBJECT: // fall through
 759     case T_OBJECT:  // fall through
 760     case T_ARRAY:
 761       if (c->as_jobject() == NULL) {
 762         if (UseCompressedOops && !wide) {
 763           __ movl(as_Address(addr), NULL_WORD);
 764         } else {
 765 #ifdef _LP64
 766           __ xorptr(rscratch1, rscratch1);
 767           null_check_here = code_offset();
 768           __ movptr(as_Address(addr), rscratch1);
 769 #else
 770           __ movptr(as_Address(addr), NULL_WORD);
 771 #endif
 772         }
 773       } else {
 774         if (is_literal_address(addr)) {
 775           ShouldNotReachHere();
 776           __ movoop(as_Address(addr, noreg), c->as_jobject(), rscratch1);
 777         } else {
 778 #ifdef _LP64
 779           __ movoop(rscratch1, c->as_jobject());
 780           if (UseCompressedOops && !wide) {
 781             __ encode_heap_oop(rscratch1);
 782             null_check_here = code_offset();
 783             __ movl(as_Address_lo(addr), rscratch1);
 784           } else {
 785             null_check_here = code_offset();
 786             __ movptr(as_Address_lo(addr), rscratch1);
 787           }
 788 #else
 789           __ movoop(as_Address(addr), c->as_jobject(), noreg);
 790 #endif
 791         }
 792       }
 793       break;
 794 
 795     case T_LONG:    // fall through
 796     case T_DOUBLE:
 797 #ifdef _LP64
 798       if (is_literal_address(addr)) {
 799         ShouldNotReachHere();
 800         __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
 801       } else {
 802         __ movptr(r10, (intptr_t)c->as_jlong_bits());
 803         null_check_here = code_offset();
 804         __ movptr(as_Address_lo(addr), r10);
 805       }
 806 #else
 807       // Always reachable in 32bit so this doesn't produce useless move literal
 808       __ movptr(as_Address_hi(addr), c->as_jint_hi_bits());
 809       __ movptr(as_Address_lo(addr), c->as_jint_lo_bits());
 810 #endif // _LP64
 811       break;
 812 
 813     case T_BOOLEAN: // fall through
 814     case T_BYTE:
 815       __ movb(as_Address(addr), c->as_jint() & 0xFF);
 816       break;
 817 
 818     case T_CHAR:    // fall through
 819     case T_SHORT:
 820       __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
 821       break;
 822 
 823     default:
 824       ShouldNotReachHere();
 825   };
 826 
 827   if (info != NULL) {
 828     add_debug_info_for_null_check(null_check_here, info);
 829   }
 830 }
 831 
 832 
 833 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 834   assert(src->is_register(), "should not call otherwise");
 835   assert(dest->is_register(), "should not call otherwise");
 836 
 837   // move between cpu-registers
 838   if (dest->is_single_cpu()) {
 839 #ifdef _LP64
 840     if (src->type() == T_LONG) {
 841       // Can do LONG -> OBJECT
 842       move_regs(src->as_register_lo(), dest->as_register());
 843       return;
 844     }
 845 #endif
 846     assert(src->is_single_cpu(), "must match");
 847     if (src->type() == T_OBJECT || src->type() == T_PRIMITIVE_OBJECT) {
 848       __ verify_oop(src->as_register());
 849     }
 850     move_regs(src->as_register(), dest->as_register());
 851 
 852   } else if (dest->is_double_cpu()) {
 853 #ifdef _LP64
 854     if (is_reference_type(src->type())) {
 855       // Surprising to me but we can see move of a long to t_object
 856       __ verify_oop(src->as_register());
 857       move_regs(src->as_register(), dest->as_register_lo());
 858       return;
 859     }
 860 #endif
 861     assert(src->is_double_cpu(), "must match");
 862     Register f_lo = src->as_register_lo();
 863     Register f_hi = src->as_register_hi();
 864     Register t_lo = dest->as_register_lo();
 865     Register t_hi = dest->as_register_hi();
 866 #ifdef _LP64
 867     assert(f_hi == f_lo, "must be same");
 868     assert(t_hi == t_lo, "must be same");
 869     move_regs(f_lo, t_lo);
 870 #else
 871     assert(f_lo != f_hi && t_lo != t_hi, "invalid register allocation");
 872 
 873 
 874     if (f_lo == t_hi && f_hi == t_lo) {
 875       swap_reg(f_lo, f_hi);
 876     } else if (f_hi == t_lo) {
 877       assert(f_lo != t_hi, "overwriting register");
 878       move_regs(f_hi, t_hi);
 879       move_regs(f_lo, t_lo);
 880     } else {
 881       assert(f_hi != t_lo, "overwriting register");
 882       move_regs(f_lo, t_lo);
 883       move_regs(f_hi, t_hi);
 884     }
 885 #endif // LP64
 886 
 887 #ifndef _LP64
 888     // special moves from fpu-register to xmm-register
 889     // necessary for method results
 890   } else if (src->is_single_xmm() && !dest->is_single_xmm()) {
 891     __ movflt(Address(rsp, 0), src->as_xmm_float_reg());
 892     __ fld_s(Address(rsp, 0));
 893   } else if (src->is_double_xmm() && !dest->is_double_xmm()) {
 894     __ movdbl(Address(rsp, 0), src->as_xmm_double_reg());
 895     __ fld_d(Address(rsp, 0));
 896   } else if (dest->is_single_xmm() && !src->is_single_xmm()) {
 897     __ fstp_s(Address(rsp, 0));
 898     __ movflt(dest->as_xmm_float_reg(), Address(rsp, 0));
 899   } else if (dest->is_double_xmm() && !src->is_double_xmm()) {
 900     __ fstp_d(Address(rsp, 0));
 901     __ movdbl(dest->as_xmm_double_reg(), Address(rsp, 0));
 902 #endif // !_LP64
 903 
 904     // move between xmm-registers
 905   } else if (dest->is_single_xmm()) {
 906     assert(src->is_single_xmm(), "must match");
 907     __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
 908   } else if (dest->is_double_xmm()) {
 909     assert(src->is_double_xmm(), "must match");
 910     __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
 911 
 912 #ifndef _LP64
 913     // move between fpu-registers (no instruction necessary because of fpu-stack)
 914   } else if (dest->is_single_fpu() || dest->is_double_fpu()) {
 915     assert(src->is_single_fpu() || src->is_double_fpu(), "must match");
 916     assert(src->fpu() == dest->fpu(), "currently should be nothing to do");
 917 #endif // !_LP64
 918 
 919   } else {
 920     ShouldNotReachHere();
 921   }
 922 }
 923 
 924 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 925   assert(src->is_register(), "should not call otherwise");
 926   assert(dest->is_stack(), "should not call otherwise");
 927 
 928   if (src->is_single_cpu()) {
 929     Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
 930     if (is_reference_type(type)) {
 931       __ verify_oop(src->as_register());
 932       __ movptr (dst, src->as_register());
 933     } else if (type == T_METADATA || type == T_ADDRESS) {
 934       __ movptr (dst, src->as_register());
 935     } else {
 936       __ movl (dst, src->as_register());
 937     }
 938 
 939   } else if (src->is_double_cpu()) {
 940     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 941     Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
 942     __ movptr (dstLO, src->as_register_lo());
 943     NOT_LP64(__ movptr (dstHI, src->as_register_hi()));
 944 
 945   } else if (src->is_single_xmm()) {
 946     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 947     __ movflt(dst_addr, src->as_xmm_float_reg());
 948 
 949   } else if (src->is_double_xmm()) {
 950     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 951     __ movdbl(dst_addr, src->as_xmm_double_reg());
 952 
 953 #ifndef _LP64
 954   } else if (src->is_single_fpu()) {
 955     assert(src->fpu_regnr() == 0, "argument must be on TOS");
 956     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 957     if (pop_fpu_stack)     __ fstp_s (dst_addr);
 958     else                   __ fst_s  (dst_addr);
 959 
 960   } else if (src->is_double_fpu()) {
 961     assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
 962     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 963     if (pop_fpu_stack)     __ fstp_d (dst_addr);
 964     else                   __ fst_d  (dst_addr);
 965 #endif // !_LP64
 966 
 967   } else {
 968     ShouldNotReachHere();
 969   }
 970 }
 971 
 972 
 973 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
 974   LIR_Address* to_addr = dest->as_address_ptr();
 975   PatchingStub* patch = NULL;
 976   Register compressed_src = rscratch1;
 977 
 978   if (is_reference_type(type)) {
 979     __ verify_oop(src->as_register());
 980 #ifdef _LP64
 981     if (UseCompressedOops && !wide) {
 982       __ movptr(compressed_src, src->as_register());
 983       __ encode_heap_oop(compressed_src);
 984       if (patch_code != lir_patch_none) {
 985         info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
 986       }
 987     }
 988 #endif
 989   }
 990 
 991   if (patch_code != lir_patch_none) {
 992     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 993     Address toa = as_Address(to_addr);
 994     assert(toa.disp() != 0, "must have");
 995   }
 996 
 997   int null_check_here = code_offset();
 998   switch (type) {
 999     case T_FLOAT: {
1000 #ifdef _LP64
1001       assert(src->is_single_xmm(), "not a float");
1002       __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
1003 #else
1004       if (src->is_single_xmm()) {
1005         __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
1006       } else {
1007         assert(src->is_single_fpu(), "must be");
1008         assert(src->fpu_regnr() == 0, "argument must be on TOS");
1009         if (pop_fpu_stack)      __ fstp_s(as_Address(to_addr));
1010         else                    __ fst_s (as_Address(to_addr));
1011       }
1012 #endif // _LP64
1013       break;
1014     }
1015 
1016     case T_DOUBLE: {
1017 #ifdef _LP64
1018       assert(src->is_double_xmm(), "not a double");
1019       __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
1020 #else
1021       if (src->is_double_xmm()) {
1022         __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
1023       } else {
1024         assert(src->is_double_fpu(), "must be");
1025         assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
1026         if (pop_fpu_stack)      __ fstp_d(as_Address(to_addr));
1027         else                    __ fst_d (as_Address(to_addr));
1028       }
1029 #endif // _LP64
1030       break;
1031     }
1032 
1033     case T_PRIMITIVE_OBJECT: // fall through
1034     case T_ARRAY:   // fall through
1035     case T_OBJECT:  // fall through
1036       if (UseCompressedOops && !wide) {
1037         __ movl(as_Address(to_addr), compressed_src);
1038       } else {
1039         __ movptr(as_Address(to_addr), src->as_register());
1040       }
1041       break;
1042     case T_METADATA:
1043       // We get here to store a method pointer to the stack to pass to
1044       // a dtrace runtime call. This can't work on 64 bit with
1045       // compressed klass ptrs: T_METADATA can be a compressed klass
1046       // ptr or a 64 bit method pointer.
1047       LP64_ONLY(ShouldNotReachHere());
1048       __ movptr(as_Address(to_addr), src->as_register());
1049       break;
1050     case T_ADDRESS:
1051       __ movptr(as_Address(to_addr), src->as_register());
1052       break;
1053     case T_INT:
1054       __ movl(as_Address(to_addr), src->as_register());
1055       break;
1056 
1057     case T_LONG: {
1058       Register from_lo = src->as_register_lo();
1059       Register from_hi = src->as_register_hi();
1060 #ifdef _LP64
1061       __ movptr(as_Address_lo(to_addr), from_lo);
1062 #else
1063       Register base = to_addr->base()->as_register();
1064       Register index = noreg;
1065       if (to_addr->index()->is_register()) {
1066         index = to_addr->index()->as_register();
1067       }
1068       if (base == from_lo || index == from_lo) {
1069         assert(base != from_hi, "can't be");
1070         assert(index == noreg || (index != base && index != from_hi), "can't handle this");
1071         __ movl(as_Address_hi(to_addr), from_hi);
1072         if (patch != NULL) {
1073           patching_epilog(patch, lir_patch_high, base, info);
1074           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1075           patch_code = lir_patch_low;
1076         }
1077         __ movl(as_Address_lo(to_addr), from_lo);
1078       } else {
1079         assert(index == noreg || (index != base && index != from_lo), "can't handle this");
1080         __ movl(as_Address_lo(to_addr), from_lo);
1081         if (patch != NULL) {
1082           patching_epilog(patch, lir_patch_low, base, info);
1083           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1084           patch_code = lir_patch_high;
1085         }
1086         __ movl(as_Address_hi(to_addr), from_hi);
1087       }
1088 #endif // _LP64
1089       break;
1090     }
1091 
1092     case T_BYTE:    // fall through
1093     case T_BOOLEAN: {
1094       Register src_reg = src->as_register();
1095       Address dst_addr = as_Address(to_addr);
1096       assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
1097       __ movb(dst_addr, src_reg);
1098       break;
1099     }
1100 
1101     case T_CHAR:    // fall through
1102     case T_SHORT:
1103       __ movw(as_Address(to_addr), src->as_register());
1104       break;
1105 
1106     default:
1107       ShouldNotReachHere();
1108   }
1109   if (info != NULL) {
1110     add_debug_info_for_null_check(null_check_here, info);
1111   }
1112 
1113   if (patch_code != lir_patch_none) {
1114     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
1115   }
1116 }
1117 
1118 
1119 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1120   assert(src->is_stack(), "should not call otherwise");
1121   assert(dest->is_register(), "should not call otherwise");
1122 
1123   if (dest->is_single_cpu()) {
1124     if (is_reference_type(type)) {
1125       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1126       __ verify_oop(dest->as_register());
1127     } else if (type == T_METADATA || type == T_ADDRESS) {
1128       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1129     } else {
1130       __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1131     }
1132 
1133   } else if (dest->is_double_cpu()) {
1134     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
1135     Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
1136     __ movptr(dest->as_register_lo(), src_addr_LO);
1137     NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI));
1138 
1139   } else if (dest->is_single_xmm()) {
1140     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1141     __ movflt(dest->as_xmm_float_reg(), src_addr);
1142 
1143   } else if (dest->is_double_xmm()) {
1144     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1145     __ movdbl(dest->as_xmm_double_reg(), src_addr);
1146 
1147 #ifndef _LP64
1148   } else if (dest->is_single_fpu()) {
1149     assert(dest->fpu_regnr() == 0, "dest must be TOS");
1150     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1151     __ fld_s(src_addr);
1152 
1153   } else if (dest->is_double_fpu()) {
1154     assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1155     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1156     __ fld_d(src_addr);
1157 #endif // _LP64
1158 
1159   } else {
1160     ShouldNotReachHere();
1161   }
1162 }
1163 
1164 
1165 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1166   if (src->is_single_stack()) {
1167     if (is_reference_type(type)) {
1168       __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
1169       __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
1170     } else {
1171 #ifndef _LP64
1172       __ pushl(frame_map()->address_for_slot(src ->single_stack_ix()));
1173       __ popl (frame_map()->address_for_slot(dest->single_stack_ix()));
1174 #else
1175       //no pushl on 64bits
1176       __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
1177       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
1178 #endif
1179     }
1180 
1181   } else if (src->is_double_stack()) {
1182 #ifdef _LP64
1183     __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
1184     __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
1185 #else
1186     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0));
1187     // push and pop the part at src + wordSize, adding wordSize for the previous push
1188     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize));
1189     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize));
1190     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0));
1191 #endif // _LP64
1192 
1193   } else {
1194     ShouldNotReachHere();
1195   }
1196 }
1197 
1198 
1199 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
1200   assert(src->is_address(), "should not call otherwise");
1201   assert(dest->is_register(), "should not call otherwise");
1202 
1203   LIR_Address* addr = src->as_address_ptr();
1204   Address from_addr = as_Address(addr);
1205 
1206   if (addr->base()->type() == T_OBJECT || addr->base()->type() == T_PRIMITIVE_OBJECT) {
1207     __ verify_oop(addr->base()->as_pointer_register());
1208   }
1209 
1210   switch (type) {
1211     case T_BOOLEAN: // fall through
1212     case T_BYTE:    // fall through
1213     case T_CHAR:    // fall through
1214     case T_SHORT:
1215       if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1216         // on pre P6 processors we may get partial register stalls
1217         // so blow away the value of to_rinfo before loading a
1218         // partial word into it.  Do it here so that it precedes
1219         // the potential patch point below.
1220         __ xorptr(dest->as_register(), dest->as_register());
1221       }
1222       break;
1223    default:
1224      break;
1225   }
1226 
1227   PatchingStub* patch = NULL;
1228   if (patch_code != lir_patch_none) {
1229     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1230     assert(from_addr.disp() != 0, "must have");
1231   }
1232   if (info != NULL) {
1233     add_debug_info_for_null_check_here(info);
1234   }
1235 
1236   switch (type) {
1237     case T_FLOAT: {
1238       if (dest->is_single_xmm()) {
1239         __ movflt(dest->as_xmm_float_reg(), from_addr);
1240       } else {
1241 #ifndef _LP64
1242         assert(dest->is_single_fpu(), "must be");
1243         assert(dest->fpu_regnr() == 0, "dest must be TOS");
1244         __ fld_s(from_addr);
1245 #else
1246         ShouldNotReachHere();
1247 #endif // !LP64
1248       }
1249       break;
1250     }
1251 
1252     case T_DOUBLE: {
1253       if (dest->is_double_xmm()) {
1254         __ movdbl(dest->as_xmm_double_reg(), from_addr);
1255       } else {
1256 #ifndef _LP64
1257         assert(dest->is_double_fpu(), "must be");
1258         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1259         __ fld_d(from_addr);
1260 #else
1261         ShouldNotReachHere();
1262 #endif // !LP64
1263       }
1264       break;
1265     }
1266 
1267     case T_PRIMITIVE_OBJECT: // fall through
1268     case T_OBJECT:  // fall through
1269     case T_ARRAY:   // fall through
1270       if (UseCompressedOops && !wide) {
1271         __ movl(dest->as_register(), from_addr);
1272       } else {
1273         __ movptr(dest->as_register(), from_addr);
1274       }
1275       break;
1276 
1277     case T_ADDRESS:
1278       __ movptr(dest->as_register(), from_addr);
1279       break;
1280     case T_INT:
1281       __ movl(dest->as_register(), from_addr);
1282       break;
1283 
1284     case T_LONG: {
1285       Register to_lo = dest->as_register_lo();
1286       Register to_hi = dest->as_register_hi();
1287 #ifdef _LP64
1288       __ movptr(to_lo, as_Address_lo(addr));
1289 #else
1290       Register base = addr->base()->as_register();
1291       Register index = noreg;
1292       if (addr->index()->is_register()) {
1293         index = addr->index()->as_register();
1294       }
1295       if ((base == to_lo && index == to_hi) ||
1296           (base == to_hi && index == to_lo)) {
1297         // addresses with 2 registers are only formed as a result of
1298         // array access so this code will never have to deal with
1299         // patches or null checks.
1300         assert(info == NULL && patch == NULL, "must be");
1301         __ lea(to_hi, as_Address(addr));
1302         __ movl(to_lo, Address(to_hi, 0));
1303         __ movl(to_hi, Address(to_hi, BytesPerWord));
1304       } else if (base == to_lo || index == to_lo) {
1305         assert(base != to_hi, "can't be");
1306         assert(index == noreg || (index != base && index != to_hi), "can't handle this");
1307         __ movl(to_hi, as_Address_hi(addr));
1308         if (patch != NULL) {
1309           patching_epilog(patch, lir_patch_high, base, info);
1310           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1311           patch_code = lir_patch_low;
1312         }
1313         __ movl(to_lo, as_Address_lo(addr));
1314       } else {
1315         assert(index == noreg || (index != base && index != to_lo), "can't handle this");
1316         __ movl(to_lo, as_Address_lo(addr));
1317         if (patch != NULL) {
1318           patching_epilog(patch, lir_patch_low, base, info);
1319           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1320           patch_code = lir_patch_high;
1321         }
1322         __ movl(to_hi, as_Address_hi(addr));
1323       }
1324 #endif // _LP64
1325       break;
1326     }
1327 
1328     case T_BOOLEAN: // fall through
1329     case T_BYTE: {
1330       Register dest_reg = dest->as_register();
1331       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1332       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1333         __ movsbl(dest_reg, from_addr);
1334       } else {
1335         __ movb(dest_reg, from_addr);
1336         __ shll(dest_reg, 24);
1337         __ sarl(dest_reg, 24);
1338       }
1339       break;
1340     }
1341 
1342     case T_CHAR: {
1343       Register dest_reg = dest->as_register();
1344       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1345       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1346         __ movzwl(dest_reg, from_addr);
1347       } else {
1348         __ movw(dest_reg, from_addr);
1349       }
1350       break;
1351     }
1352 
1353     case T_SHORT: {
1354       Register dest_reg = dest->as_register();
1355       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1356         __ movswl(dest_reg, from_addr);
1357       } else {
1358         __ movw(dest_reg, from_addr);
1359         __ shll(dest_reg, 16);
1360         __ sarl(dest_reg, 16);
1361       }
1362       break;
1363     }
1364 
1365     default:
1366       ShouldNotReachHere();
1367   }
1368 
1369   if (patch != NULL) {
1370     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1371   }
1372 
1373   if (is_reference_type(type)) {
1374 #ifdef _LP64
1375     if (UseCompressedOops && !wide) {
1376       __ decode_heap_oop(dest->as_register());
1377     }
1378 #endif
1379 
1380     // Load barrier has not yet been applied, so ZGC can't verify the oop here
1381     if (!UseZGC) {
1382       __ verify_oop(dest->as_register());
1383     }
1384   }
1385 }
1386 
1387 
1388 NEEDS_CLEANUP; // This could be static?
1389 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1390   int elem_size = type2aelembytes(type);
1391   switch (elem_size) {
1392     case 1: return Address::times_1;
1393     case 2: return Address::times_2;
1394     case 4: return Address::times_4;
1395     case 8: return Address::times_8;
1396   }
1397   ShouldNotReachHere();
1398   return Address::no_scale;
1399 }
1400 
1401 
1402 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1403   switch (op->code()) {
1404     case lir_idiv:
1405     case lir_irem:
1406       arithmetic_idiv(op->code(),
1407                       op->in_opr1(),
1408                       op->in_opr2(),
1409                       op->in_opr3(),
1410                       op->result_opr(),
1411                       op->info());
1412       break;
1413     case lir_fmad:
1414       __ fmad(op->result_opr()->as_xmm_double_reg(),
1415               op->in_opr1()->as_xmm_double_reg(),
1416               op->in_opr2()->as_xmm_double_reg(),
1417               op->in_opr3()->as_xmm_double_reg());
1418       break;
1419     case lir_fmaf:
1420       __ fmaf(op->result_opr()->as_xmm_float_reg(),
1421               op->in_opr1()->as_xmm_float_reg(),
1422               op->in_opr2()->as_xmm_float_reg(),
1423               op->in_opr3()->as_xmm_float_reg());
1424       break;
1425     default:      ShouldNotReachHere(); break;
1426   }
1427 }
1428 
1429 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1430 #ifdef ASSERT
1431   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1432   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1433   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1434 #endif
1435 
1436   if (op->cond() == lir_cond_always) {
1437     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1438     __ jmp (*(op->label()));
1439   } else {
1440     Assembler::Condition acond = Assembler::zero;
1441     if (op->code() == lir_cond_float_branch) {
1442       assert(op->ublock() != NULL, "must have unordered successor");
1443       __ jcc(Assembler::parity, *(op->ublock()->label()));
1444       switch(op->cond()) {
1445         case lir_cond_equal:        acond = Assembler::equal;      break;
1446         case lir_cond_notEqual:     acond = Assembler::notEqual;   break;
1447         case lir_cond_less:         acond = Assembler::below;      break;
1448         case lir_cond_lessEqual:    acond = Assembler::belowEqual; break;
1449         case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1450         case lir_cond_greater:      acond = Assembler::above;      break;
1451         default:                         ShouldNotReachHere();
1452       }
1453     } else {
1454       switch (op->cond()) {
1455         case lir_cond_equal:        acond = Assembler::equal;       break;
1456         case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
1457         case lir_cond_less:         acond = Assembler::less;        break;
1458         case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
1459         case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1460         case lir_cond_greater:      acond = Assembler::greater;     break;
1461         case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
1462         case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
1463         default:                         ShouldNotReachHere();
1464       }
1465     }
1466     __ jcc(acond,*(op->label()));
1467   }
1468 }
1469 
1470 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1471   LIR_Opr src  = op->in_opr();
1472   LIR_Opr dest = op->result_opr();
1473 
1474   switch (op->bytecode()) {
1475     case Bytecodes::_i2l:
1476 #ifdef _LP64
1477       __ movl2ptr(dest->as_register_lo(), src->as_register());
1478 #else
1479       move_regs(src->as_register(), dest->as_register_lo());
1480       move_regs(src->as_register(), dest->as_register_hi());
1481       __ sarl(dest->as_register_hi(), 31);
1482 #endif // LP64
1483       break;
1484 
1485     case Bytecodes::_l2i:
1486 #ifdef _LP64
1487       __ movl(dest->as_register(), src->as_register_lo());
1488 #else
1489       move_regs(src->as_register_lo(), dest->as_register());
1490 #endif
1491       break;
1492 
1493     case Bytecodes::_i2b:
1494       move_regs(src->as_register(), dest->as_register());
1495       __ sign_extend_byte(dest->as_register());
1496       break;
1497 
1498     case Bytecodes::_i2c:
1499       move_regs(src->as_register(), dest->as_register());
1500       __ andl(dest->as_register(), 0xFFFF);
1501       break;
1502 
1503     case Bytecodes::_i2s:
1504       move_regs(src->as_register(), dest->as_register());
1505       __ sign_extend_short(dest->as_register());
1506       break;
1507 
1508 
1509 #ifdef _LP64
1510     case Bytecodes::_f2d:
1511       __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1512       break;
1513 
1514     case Bytecodes::_d2f:
1515       __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1516       break;
1517 
1518     case Bytecodes::_i2f:
1519       __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1520       break;
1521 
1522     case Bytecodes::_i2d:
1523       __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1524       break;
1525 
1526     case Bytecodes::_l2f:
1527       __ cvtsi2ssq(dest->as_xmm_float_reg(), src->as_register_lo());
1528       break;
1529 
1530     case Bytecodes::_l2d:
1531       __ cvtsi2sdq(dest->as_xmm_double_reg(), src->as_register_lo());
1532       break;
1533 
1534     case Bytecodes::_f2i:
1535       __ convert_f2i(dest->as_register(), src->as_xmm_float_reg());
1536       break;
1537 
1538     case Bytecodes::_d2i:
1539       __ convert_d2i(dest->as_register(), src->as_xmm_double_reg());
1540       break;
1541 
1542     case Bytecodes::_f2l:
1543       __ convert_f2l(dest->as_register_lo(), src->as_xmm_float_reg());
1544       break;
1545 
1546     case Bytecodes::_d2l:
1547       __ convert_d2l(dest->as_register_lo(), src->as_xmm_double_reg());
1548       break;
1549 #else
1550     case Bytecodes::_f2d:
1551     case Bytecodes::_d2f:
1552       if (dest->is_single_xmm()) {
1553         __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1554       } else if (dest->is_double_xmm()) {
1555         __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1556       } else {
1557         assert(src->fpu() == dest->fpu(), "register must be equal");
1558         // do nothing (float result is rounded later through spilling)
1559       }
1560       break;
1561 
1562     case Bytecodes::_i2f:
1563     case Bytecodes::_i2d:
1564       if (dest->is_single_xmm()) {
1565         __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1566       } else if (dest->is_double_xmm()) {
1567         __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1568       } else {
1569         assert(dest->fpu() == 0, "result must be on TOS");
1570         __ movl(Address(rsp, 0), src->as_register());
1571         __ fild_s(Address(rsp, 0));
1572       }
1573       break;
1574 
1575     case Bytecodes::_l2f:
1576     case Bytecodes::_l2d:
1577       assert(!dest->is_xmm_register(), "result in xmm register not supported (no SSE instruction present)");
1578       assert(dest->fpu() == 0, "result must be on TOS");
1579       __ movptr(Address(rsp, 0),          src->as_register_lo());
1580       __ movl(Address(rsp, BytesPerWord), src->as_register_hi());
1581       __ fild_d(Address(rsp, 0));
1582       // float result is rounded later through spilling
1583       break;
1584 
1585     case Bytecodes::_f2i:
1586     case Bytecodes::_d2i:
1587       if (src->is_single_xmm()) {
1588         __ cvttss2sil(dest->as_register(), src->as_xmm_float_reg());
1589       } else if (src->is_double_xmm()) {
1590         __ cvttsd2sil(dest->as_register(), src->as_xmm_double_reg());
1591       } else {
1592         assert(src->fpu() == 0, "input must be on TOS");
1593         __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_trunc()));
1594         __ fist_s(Address(rsp, 0));
1595         __ movl(dest->as_register(), Address(rsp, 0));
1596         __ fldcw(ExternalAddress(StubRoutines::x86::addr_fpu_cntrl_wrd_std()));
1597       }
1598       // IA32 conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
1599       assert(op->stub() != NULL, "stub required");
1600       __ cmpl(dest->as_register(), 0x80000000);
1601       __ jcc(Assembler::equal, *op->stub()->entry());
1602       __ bind(*op->stub()->continuation());
1603       break;
1604 
1605     case Bytecodes::_f2l:
1606     case Bytecodes::_d2l:
1607       assert(!src->is_xmm_register(), "input in xmm register not supported (no SSE instruction present)");
1608       assert(src->fpu() == 0, "input must be on TOS");
1609       assert(dest == FrameMap::long0_opr, "runtime stub places result in these registers");
1610 
1611       // instruction sequence too long to inline it here
1612       {
1613         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::fpu2long_stub_id)));
1614       }
1615       break;
1616 #endif // _LP64
1617 
1618     default: ShouldNotReachHere();
1619   }
1620 }
1621 
1622 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1623   if (op->init_check()) {
1624     add_debug_info_for_null_check_here(op->stub()->info());
1625     __ cmpb(Address(op->klass()->as_register(),
1626                     InstanceKlass::init_state_offset()),
1627                     InstanceKlass::fully_initialized);
1628     __ jcc(Assembler::notEqual, *op->stub()->entry());
1629   }
1630   __ allocate_object(op->obj()->as_register(),
1631                      op->tmp1()->as_register(),
1632                      op->tmp2()->as_register(),
1633                      op->header_size(),
1634                      op->object_size(),
1635                      op->klass()->as_register(),
1636                      *op->stub()->entry());
1637   __ bind(*op->stub()->continuation());
1638 }
1639 
1640 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1641   Register len =  op->len()->as_register();
1642   LP64_ONLY( __ movslq(len, len); )
1643 
1644   if (UseSlowPath || op->type() == T_PRIMITIVE_OBJECT ||
1645       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1646       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1647     __ jmp(*op->stub()->entry());
1648   } else {
1649     Register tmp1 = op->tmp1()->as_register();
1650     Register tmp2 = op->tmp2()->as_register();
1651     Register tmp3 = op->tmp3()->as_register();
1652     if (len == tmp1) {
1653       tmp1 = tmp3;
1654     } else if (len == tmp2) {
1655       tmp2 = tmp3;
1656     } else if (len == tmp3) {
1657       // everything is ok
1658     } else {
1659       __ mov(tmp3, len);
1660     }
1661     __ allocate_array(op->obj()->as_register(),
1662                       len,
1663                       tmp1,
1664                       tmp2,
1665                       arrayOopDesc::header_size(op->type()),
1666                       array_element_size(op->type()),
1667                       op->klass()->as_register(),
1668                       *op->stub()->entry());
1669   }
1670   __ bind(*op->stub()->continuation());
1671 }
1672 
1673 void LIR_Assembler::type_profile_helper(Register mdo,
1674                                         ciMethodData *md, ciProfileData *data,
1675                                         Register recv, Label* update_done) {
1676   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1677     Label next_test;
1678     // See if the receiver is receiver[n].
1679     __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1680     __ jccb(Assembler::notEqual, next_test);
1681     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1682     __ addptr(data_addr, DataLayout::counter_increment);
1683     __ jmp(*update_done);
1684     __ bind(next_test);
1685   }
1686 
1687   // Didn't find receiver; find next empty slot and fill it in
1688   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1689     Label next_test;
1690     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1691     __ cmpptr(recv_addr, NULL_WORD);
1692     __ jccb(Assembler::notEqual, next_test);
1693     __ movptr(recv_addr, recv);
1694     __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1695     __ jmp(*update_done);
1696     __ bind(next_test);
1697   }
1698 }
1699 
1700 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1701   // we always need a stub for the failure case.
1702   CodeStub* stub = op->stub();
1703   Register obj = op->object()->as_register();
1704   Register k_RInfo = op->tmp1()->as_register();
1705   Register klass_RInfo = op->tmp2()->as_register();
1706   Register dst = op->result_opr()->as_register();
1707   ciKlass* k = op->klass();
1708   Register Rtmp1 = noreg;
1709   Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1710 
1711   // check if it needs to be profiled
1712   ciMethodData* md = NULL;
1713   ciProfileData* data = NULL;
1714 
1715   if (op->should_profile()) {
1716     ciMethod* method = op->profiled_method();
1717     assert(method != NULL, "Should have method");
1718     int bci = op->profiled_bci();
1719     md = method->method_data_or_null();
1720     assert(md != NULL, "Sanity");
1721     data = md->bci_to_data(bci);
1722     assert(data != NULL,                "need data for type check");
1723     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1724   }
1725   Label profile_cast_success, profile_cast_failure;
1726   Label *success_target = op->should_profile() ? &profile_cast_success : success;
1727   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1728 
1729   if (obj == k_RInfo) {
1730     k_RInfo = dst;
1731   } else if (obj == klass_RInfo) {
1732     klass_RInfo = dst;
1733   }
1734   if (k->is_loaded() && !UseCompressedClassPointers) {
1735     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1736   } else {
1737     Rtmp1 = op->tmp3()->as_register();
1738     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1739   }
1740 
1741   assert_different_registers(obj, k_RInfo, klass_RInfo);
1742 
1743   if (op->need_null_check()) {
1744     __ cmpptr(obj, NULL_WORD);
1745     if (op->should_profile()) {
1746       Label not_null;
1747       __ jccb(Assembler::notEqual, not_null);
1748       // Object is null; update MDO and exit
1749       Register mdo  = klass_RInfo;
1750       __ mov_metadata(mdo, md->constant_encoding());
1751       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1752       int header_bits = BitData::null_seen_byte_constant();
1753       __ orb(data_addr, header_bits);
1754       __ jmp(*obj_is_null);
1755       __ bind(not_null);
1756     } else {
1757       __ jcc(Assembler::equal, *obj_is_null);
1758     }
1759   }
1760 
1761   if (!k->is_loaded()) {
1762     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1763   } else {
1764 #ifdef _LP64
1765     __ mov_metadata(k_RInfo, k->constant_encoding());
1766 #endif // _LP64
1767   }
1768   __ verify_oop(obj);
1769 
1770   if (op->fast_check()) {
1771     // get object class
1772     // not a safepoint as obj null check happens earlier
1773 #ifdef _LP64
1774     if (UseCompressedClassPointers) {
1775       __ load_klass(Rtmp1, obj, tmp_load_klass);
1776       __ cmpptr(k_RInfo, Rtmp1);
1777     } else {
1778       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1779     }
1780 #else
1781     if (k->is_loaded()) {
1782       __ cmpklass(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
1783     } else {
1784       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1785     }
1786 #endif
1787     __ jcc(Assembler::notEqual, *failure_target);
1788     // successful cast, fall through to profile or jump
1789   } else {
1790     // get object class
1791     // not a safepoint as obj null check happens earlier
1792     __ load_klass(klass_RInfo, obj, tmp_load_klass);
1793     if (k->is_loaded()) {
1794       // See if we get an immediate positive hit
1795 #ifdef _LP64
1796       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1797 #else
1798       __ cmpklass(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
1799 #endif // _LP64
1800       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1801         __ jcc(Assembler::notEqual, *failure_target);
1802         // successful cast, fall through to profile or jump
1803       } else {
1804         // See if we get an immediate positive hit
1805         __ jcc(Assembler::equal, *success_target);
1806         // check for self
1807 #ifdef _LP64
1808         __ cmpptr(klass_RInfo, k_RInfo);
1809 #else
1810         __ cmpklass(klass_RInfo, k->constant_encoding());
1811 #endif // _LP64
1812         __ jcc(Assembler::equal, *success_target);
1813 
1814         __ push(klass_RInfo);
1815 #ifdef _LP64
1816         __ push(k_RInfo);
1817 #else
1818         __ pushklass(k->constant_encoding(), noreg);
1819 #endif // _LP64
1820         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1821         __ pop(klass_RInfo);
1822         __ pop(klass_RInfo);
1823         // result is a boolean
1824         __ cmpl(klass_RInfo, 0);
1825         __ jcc(Assembler::equal, *failure_target);
1826         // successful cast, fall through to profile or jump
1827       }
1828     } else {
1829       // perform the fast part of the checking logic
1830       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1831       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1832       __ push(klass_RInfo);
1833       __ push(k_RInfo);
1834       __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1835       __ pop(klass_RInfo);
1836       __ pop(k_RInfo);
1837       // result is a boolean
1838       __ cmpl(k_RInfo, 0);
1839       __ jcc(Assembler::equal, *failure_target);
1840       // successful cast, fall through to profile or jump
1841     }
1842   }
1843   if (op->should_profile()) {
1844     Register mdo  = klass_RInfo, recv = k_RInfo;
1845     __ bind(profile_cast_success);
1846     __ mov_metadata(mdo, md->constant_encoding());
1847     __ load_klass(recv, obj, tmp_load_klass);
1848     type_profile_helper(mdo, md, data, recv, success);
1849     __ jmp(*success);
1850 
1851     __ bind(profile_cast_failure);
1852     __ mov_metadata(mdo, md->constant_encoding());
1853     Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1854     __ subptr(counter_addr, DataLayout::counter_increment);
1855     __ jmp(*failure);
1856   }
1857   __ jmp(*success);
1858 }
1859 
1860 
1861 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1862   Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1863   LIR_Code code = op->code();
1864   if (code == lir_store_check) {
1865     Register value = op->object()->as_register();
1866     Register array = op->array()->as_register();
1867     Register k_RInfo = op->tmp1()->as_register();
1868     Register klass_RInfo = op->tmp2()->as_register();
1869     Register Rtmp1 = op->tmp3()->as_register();
1870 
1871     CodeStub* stub = op->stub();
1872 
1873     // check if it needs to be profiled
1874     ciMethodData* md = NULL;
1875     ciProfileData* data = NULL;
1876 
1877     if (op->should_profile()) {
1878       ciMethod* method = op->profiled_method();
1879       assert(method != NULL, "Should have method");
1880       int bci = op->profiled_bci();
1881       md = method->method_data_or_null();
1882       assert(md != NULL, "Sanity");
1883       data = md->bci_to_data(bci);
1884       assert(data != NULL,                "need data for type check");
1885       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1886     }
1887     Label profile_cast_success, profile_cast_failure, done;
1888     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1889     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1890 
1891     __ cmpptr(value, NULL_WORD);
1892     if (op->should_profile()) {
1893       Label not_null;
1894       __ jccb(Assembler::notEqual, not_null);
1895       // Object is null; update MDO and exit
1896       Register mdo  = klass_RInfo;
1897       __ mov_metadata(mdo, md->constant_encoding());
1898       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1899       int header_bits = BitData::null_seen_byte_constant();
1900       __ orb(data_addr, header_bits);
1901       __ jmp(done);
1902       __ bind(not_null);
1903     } else {
1904       __ jcc(Assembler::equal, done);
1905     }
1906 
1907     add_debug_info_for_null_check_here(op->info_for_exception());
1908     __ load_klass(k_RInfo, array, tmp_load_klass);
1909     __ load_klass(klass_RInfo, value, tmp_load_klass);
1910 
1911     // get instance klass (it's already uncompressed)
1912     __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1913     // perform the fast part of the checking logic
1914     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1915     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1916     __ push(klass_RInfo);
1917     __ push(k_RInfo);
1918     __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1919     __ pop(klass_RInfo);
1920     __ pop(k_RInfo);
1921     // result is a boolean
1922     __ cmpl(k_RInfo, 0);
1923     __ jcc(Assembler::equal, *failure_target);
1924     // fall through to the success case
1925 
1926     if (op->should_profile()) {
1927       Register mdo  = klass_RInfo, recv = k_RInfo;
1928       __ bind(profile_cast_success);
1929       __ mov_metadata(mdo, md->constant_encoding());
1930       __ load_klass(recv, value, tmp_load_klass);
1931       type_profile_helper(mdo, md, data, recv, &done);
1932       __ jmpb(done);
1933 
1934       __ bind(profile_cast_failure);
1935       __ mov_metadata(mdo, md->constant_encoding());
1936       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1937       __ subptr(counter_addr, DataLayout::counter_increment);
1938       __ jmp(*stub->entry());
1939     }
1940 
1941     __ bind(done);
1942   } else
1943     if (code == lir_checkcast) {
1944       Register obj = op->object()->as_register();
1945       Register dst = op->result_opr()->as_register();
1946       Label success;
1947       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1948       __ bind(success);
1949       if (dst != obj) {
1950         __ mov(dst, obj);
1951       }
1952     } else
1953       if (code == lir_instanceof) {
1954         Register obj = op->object()->as_register();
1955         Register dst = op->result_opr()->as_register();
1956         Label success, failure, done;
1957         emit_typecheck_helper(op, &success, &failure, &failure);
1958         __ bind(failure);
1959         __ xorptr(dst, dst);
1960         __ jmpb(done);
1961         __ bind(success);
1962         __ movptr(dst, 1);
1963         __ bind(done);
1964       } else {
1965         ShouldNotReachHere();
1966       }
1967 
1968 }
1969 
1970 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1971   // We are loading/storing from/to an array that *may* be flattened (the
1972   // declared type is Object[], abstract[], interface[] or VT.ref[]).
1973   // If this array is flattened, take the slow path.
1974   Register klass = op->tmp()->as_register();
1975   if (UseArrayMarkWordCheck) {
1976     __ test_flattened_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1977   } else {
1978     Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
1979     __ load_klass(klass, op->array()->as_register(), tmp_load_klass);
1980     __ movl(klass, Address(klass, Klass::layout_helper_offset()));
1981     __ testl(klass, Klass::_lh_array_tag_flat_value_bit_inplace);
1982     __ jcc(Assembler::notZero, *op->stub()->entry());
1983   }
1984   if (!op->value()->is_illegal()) {
1985     // The array is not flattened, but it might be null-free. If we are storing
1986     // a null into a null-free array, take the slow path (which will throw NPE).
1987     Label skip;
1988     __ cmpptr(op->value()->as_register(), NULL_WORD);
1989     __ jcc(Assembler::notEqual, skip);
1990     if (UseArrayMarkWordCheck) {
1991       __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1992     } else {
1993       __ testl(klass, Klass::_lh_null_free_array_bit_inplace);
1994       __ jcc(Assembler::notZero, *op->stub()->entry());
1995     }
1996     __ bind(skip);
1997   }
1998 }
1999 
2000 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
2001   // We are storing into an array that *may* be null-free (the declared type is
2002   // Object[], abstract[], interface[] or VT.ref[]).
2003   if (UseArrayMarkWordCheck) {
2004     Label test_mark_word;
2005     Register tmp = op->tmp()->as_register();
2006     __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
2007     __ testl(tmp, markWord::unlocked_value);
2008     __ jccb(Assembler::notZero, test_mark_word);
2009     __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
2010     __ bind(test_mark_word);
2011     __ testl(tmp, markWord::null_free_array_bit_in_place);
2012   } else {
2013     Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
2014     Register klass = op->tmp()->as_register();
2015     __ load_klass(klass, op->array()->as_register(), tmp_load_klass);
2016     __ movl(klass, Address(klass, Klass::layout_helper_offset()));
2017     __ testl(klass, Klass::_lh_null_free_array_bit_inplace);
2018   }
2019 }
2020 
2021 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
2022   Label L_oops_equal;
2023   Label L_oops_not_equal;
2024   Label L_end;
2025 
2026   Register left  = op->left()->as_register();
2027   Register right = op->right()->as_register();
2028 
2029   __ cmpptr(left, right);
2030   __ jcc(Assembler::equal, L_oops_equal);
2031 
2032   // (1) Null check -- if one of the operands is null, the other must not be null (because
2033   //     the two references are not equal), so they are not substitutable,
2034   //     FIXME: do null check only if the operand is nullable
2035   __ testptr(left, right);
2036   __ jcc(Assembler::zero, L_oops_not_equal);
2037 
2038   ciKlass* left_klass = op->left_klass();
2039   ciKlass* right_klass = op->right_klass();
2040 
2041   // (2) Inline type check -- if either of the operands is not a inline type,
2042   //     they are not substitutable. We do this only if we are not sure that the
2043   //     operands are inline type
2044   if ((left_klass == NULL || right_klass == NULL) ||// The klass is still unloaded, or came from a Phi node.
2045       !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
2046     Register tmp1  = op->tmp1()->as_register();
2047     __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
2048     __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
2049     __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
2050     __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
2051     __ jcc(Assembler::notEqual, L_oops_not_equal);
2052   }
2053 
2054   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
2055   if (left_klass != NULL && left_klass->is_inlinetype() && left_klass == right_klass) {
2056     // No need to load klass -- the operands are statically known to be the same inline klass.
2057     __ jmp(*op->stub()->entry());
2058   } else {
2059     Register left_klass_op = op->left_klass_op()->as_register();
2060     Register right_klass_op = op->right_klass_op()->as_register();
2061 
2062     if (UseCompressedClassPointers) {
2063       __ movl(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
2064       __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2065       __ cmpl(left_klass_op, right_klass_op);
2066     } else {
2067       __ movptr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
2068       __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2069       __ cmpptr(left_klass_op, right_klass_op);
2070     }
2071 
2072     __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
2073     // fall through to L_oops_not_equal
2074   }
2075 
2076   __ bind(L_oops_not_equal);
2077   move(op->not_equal_result(), op->result_opr());
2078   __ jmp(L_end);
2079 
2080   __ bind(L_oops_equal);
2081   move(op->equal_result(), op->result_opr());
2082   __ jmp(L_end);
2083 
2084   // We've returned from the stub. RAX contains 0x0 IFF the two
2085   // operands are not substitutable. (Don't compare against 0x1 in case the
2086   // C compiler is naughty)
2087   __ bind(*op->stub()->continuation());
2088   __ cmpl(rax, 0);
2089   __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
2090   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
2091   // fall-through
2092   __ bind(L_end);
2093 }
2094 
2095 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2096   if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) {
2097     assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
2098     assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
2099     assert(op->new_value()->as_register_lo() == rbx, "wrong register");
2100     assert(op->new_value()->as_register_hi() == rcx, "wrong register");
2101     Register addr = op->addr()->as_register();
2102     __ lock();
2103     NOT_LP64(__ cmpxchg8(Address(addr, 0)));
2104 
2105   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
2106     NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
2107     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2108     Register newval = op->new_value()->as_register();
2109     Register cmpval = op->cmp_value()->as_register();
2110     assert(cmpval == rax, "wrong register");
2111     assert(newval != noreg, "new val must be register");
2112     assert(cmpval != newval, "cmp and new values must be in different registers");
2113     assert(cmpval != addr, "cmp and addr must be in different registers");
2114     assert(newval != addr, "new value and addr must be in different registers");
2115 
2116     if ( op->code() == lir_cas_obj) {
2117 #ifdef _LP64
2118       if (UseCompressedOops) {
2119         __ encode_heap_oop(cmpval);
2120         __ mov(rscratch1, newval);
2121         __ encode_heap_oop(rscratch1);
2122         __ lock();
2123         // cmpval (rax) is implicitly used by this instruction
2124         __ cmpxchgl(rscratch1, Address(addr, 0));
2125       } else
2126 #endif
2127       {
2128         __ lock();
2129         __ cmpxchgptr(newval, Address(addr, 0));
2130       }
2131     } else {
2132       assert(op->code() == lir_cas_int, "lir_cas_int expected");
2133       __ lock();
2134       __ cmpxchgl(newval, Address(addr, 0));
2135     }
2136 #ifdef _LP64
2137   } else if (op->code() == lir_cas_long) {
2138     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2139     Register newval = op->new_value()->as_register_lo();
2140     Register cmpval = op->cmp_value()->as_register_lo();
2141     assert(cmpval == rax, "wrong register");
2142     assert(newval != noreg, "new val must be register");
2143     assert(cmpval != newval, "cmp and new values must be in different registers");
2144     assert(cmpval != addr, "cmp and addr must be in different registers");
2145     assert(newval != addr, "new value and addr must be in different registers");
2146     __ lock();
2147     __ cmpxchgq(newval, Address(addr, 0));
2148 #endif // _LP64
2149   } else {
2150     Unimplemented();
2151   }
2152 }
2153 
2154 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
2155   assert(dst->is_cpu_register(), "must be");
2156   assert(dst->type() == src->type(), "must be");
2157 
2158   if (src->is_cpu_register()) {
2159     reg2reg(src, dst);
2160   } else if (src->is_stack()) {
2161     stack2reg(src, dst, dst->type());
2162   } else if (src->is_constant()) {
2163     const2reg(src, dst, lir_patch_none, NULL);
2164   } else {
2165     ShouldNotReachHere();
2166   }
2167 }
2168 
2169 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
2170                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
2171   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
2172 
2173   Assembler::Condition acond, ncond;
2174   switch (condition) {
2175     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
2176     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
2177     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
2178     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
2179     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
2180     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
2181     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
2182     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
2183     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
2184                                 ShouldNotReachHere();
2185   }
2186 
2187   if (opr1->is_cpu_register()) {
2188     reg2reg(opr1, result);
2189   } else if (opr1->is_stack()) {
2190     stack2reg(opr1, result, result->type());
2191   } else if (opr1->is_constant()) {
2192     const2reg(opr1, result, lir_patch_none, NULL);
2193   } else {
2194     ShouldNotReachHere();
2195   }
2196 
2197   if (VM_Version::supports_cmov() && !opr2->is_constant()) {
2198     // optimized version that does not require a branch
2199     if (opr2->is_single_cpu()) {
2200       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
2201       __ cmov(ncond, result->as_register(), opr2->as_register());
2202     } else if (opr2->is_double_cpu()) {
2203       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2204       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2205       __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
2206       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());)
2207     } else if (opr2->is_single_stack()) {
2208       __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
2209     } else if (opr2->is_double_stack()) {
2210       __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
2211       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));)
2212     } else {
2213       ShouldNotReachHere();
2214     }
2215 
2216   } else {
2217     Label skip;
2218     __ jcc (acond, skip);
2219     if (opr2->is_cpu_register()) {
2220       reg2reg(opr2, result);
2221     } else if (opr2->is_stack()) {
2222       stack2reg(opr2, result, result->type());
2223     } else if (opr2->is_constant()) {
2224       const2reg(opr2, result, lir_patch_none, NULL);
2225     } else {
2226       ShouldNotReachHere();
2227     }
2228     __ bind(skip);
2229   }
2230 }
2231 
2232 
2233 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
2234   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
2235 
2236   if (left->is_single_cpu()) {
2237     assert(left == dest, "left and dest must be equal");
2238     Register lreg = left->as_register();
2239 
2240     if (right->is_single_cpu()) {
2241       // cpu register - cpu register
2242       Register rreg = right->as_register();
2243       switch (code) {
2244         case lir_add: __ addl (lreg, rreg); break;
2245         case lir_sub: __ subl (lreg, rreg); break;
2246         case lir_mul: __ imull(lreg, rreg); break;
2247         default:      ShouldNotReachHere();
2248       }
2249 
2250     } else if (right->is_stack()) {
2251       // cpu register - stack
2252       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2253       switch (code) {
2254         case lir_add: __ addl(lreg, raddr); break;
2255         case lir_sub: __ subl(lreg, raddr); break;
2256         default:      ShouldNotReachHere();
2257       }
2258 
2259     } else if (right->is_constant()) {
2260       // cpu register - constant
2261       jint c = right->as_constant_ptr()->as_jint();
2262       switch (code) {
2263         case lir_add: {
2264           __ incrementl(lreg, c);
2265           break;
2266         }
2267         case lir_sub: {
2268           __ decrementl(lreg, c);
2269           break;
2270         }
2271         default: ShouldNotReachHere();
2272       }
2273 
2274     } else {
2275       ShouldNotReachHere();
2276     }
2277 
2278   } else if (left->is_double_cpu()) {
2279     assert(left == dest, "left and dest must be equal");
2280     Register lreg_lo = left->as_register_lo();
2281     Register lreg_hi = left->as_register_hi();
2282 
2283     if (right->is_double_cpu()) {
2284       // cpu register - cpu register
2285       Register rreg_lo = right->as_register_lo();
2286       Register rreg_hi = right->as_register_hi();
2287       NOT_LP64(assert_different_registers(lreg_lo, lreg_hi, rreg_lo, rreg_hi));
2288       LP64_ONLY(assert_different_registers(lreg_lo, rreg_lo));
2289       switch (code) {
2290         case lir_add:
2291           __ addptr(lreg_lo, rreg_lo);
2292           NOT_LP64(__ adcl(lreg_hi, rreg_hi));
2293           break;
2294         case lir_sub:
2295           __ subptr(lreg_lo, rreg_lo);
2296           NOT_LP64(__ sbbl(lreg_hi, rreg_hi));
2297           break;
2298         case lir_mul:
2299 #ifdef _LP64
2300           __ imulq(lreg_lo, rreg_lo);
2301 #else
2302           assert(lreg_lo == rax && lreg_hi == rdx, "must be");
2303           __ imull(lreg_hi, rreg_lo);
2304           __ imull(rreg_hi, lreg_lo);
2305           __ addl (rreg_hi, lreg_hi);
2306           __ mull (rreg_lo);
2307           __ addl (lreg_hi, rreg_hi);
2308 #endif // _LP64
2309           break;
2310         default:
2311           ShouldNotReachHere();
2312       }
2313 
2314     } else if (right->is_constant()) {
2315       // cpu register - constant
2316 #ifdef _LP64
2317       jlong c = right->as_constant_ptr()->as_jlong_bits();
2318       __ movptr(r10, (intptr_t) c);
2319       switch (code) {
2320         case lir_add:
2321           __ addptr(lreg_lo, r10);
2322           break;
2323         case lir_sub:
2324           __ subptr(lreg_lo, r10);
2325           break;
2326         default:
2327           ShouldNotReachHere();
2328       }
2329 #else
2330       jint c_lo = right->as_constant_ptr()->as_jint_lo();
2331       jint c_hi = right->as_constant_ptr()->as_jint_hi();
2332       switch (code) {
2333         case lir_add:
2334           __ addptr(lreg_lo, c_lo);
2335           __ adcl(lreg_hi, c_hi);
2336           break;
2337         case lir_sub:
2338           __ subptr(lreg_lo, c_lo);
2339           __ sbbl(lreg_hi, c_hi);
2340           break;
2341         default:
2342           ShouldNotReachHere();
2343       }
2344 #endif // _LP64
2345 
2346     } else {
2347       ShouldNotReachHere();
2348     }
2349 
2350   } else if (left->is_single_xmm()) {
2351     assert(left == dest, "left and dest must be equal");
2352     XMMRegister lreg = left->as_xmm_float_reg();
2353 
2354     if (right->is_single_xmm()) {
2355       XMMRegister rreg = right->as_xmm_float_reg();
2356       switch (code) {
2357         case lir_add: __ addss(lreg, rreg);  break;
2358         case lir_sub: __ subss(lreg, rreg);  break;
2359         case lir_mul: __ mulss(lreg, rreg);  break;
2360         case lir_div: __ divss(lreg, rreg);  break;
2361         default: ShouldNotReachHere();
2362       }
2363     } else {
2364       Address raddr;
2365       if (right->is_single_stack()) {
2366         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2367       } else if (right->is_constant()) {
2368         // hack for now
2369         raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
2370       } else {
2371         ShouldNotReachHere();
2372       }
2373       switch (code) {
2374         case lir_add: __ addss(lreg, raddr);  break;
2375         case lir_sub: __ subss(lreg, raddr);  break;
2376         case lir_mul: __ mulss(lreg, raddr);  break;
2377         case lir_div: __ divss(lreg, raddr);  break;
2378         default: ShouldNotReachHere();
2379       }
2380     }
2381 
2382   } else if (left->is_double_xmm()) {
2383     assert(left == dest, "left and dest must be equal");
2384 
2385     XMMRegister lreg = left->as_xmm_double_reg();
2386     if (right->is_double_xmm()) {
2387       XMMRegister rreg = right->as_xmm_double_reg();
2388       switch (code) {
2389         case lir_add: __ addsd(lreg, rreg);  break;
2390         case lir_sub: __ subsd(lreg, rreg);  break;
2391         case lir_mul: __ mulsd(lreg, rreg);  break;
2392         case lir_div: __ divsd(lreg, rreg);  break;
2393         default: ShouldNotReachHere();
2394       }
2395     } else {
2396       Address raddr;
2397       if (right->is_double_stack()) {
2398         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2399       } else if (right->is_constant()) {
2400         // hack for now
2401         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2402       } else {
2403         ShouldNotReachHere();
2404       }
2405       switch (code) {
2406         case lir_add: __ addsd(lreg, raddr);  break;
2407         case lir_sub: __ subsd(lreg, raddr);  break;
2408         case lir_mul: __ mulsd(lreg, raddr);  break;
2409         case lir_div: __ divsd(lreg, raddr);  break;
2410         default: ShouldNotReachHere();
2411       }
2412     }
2413 
2414 #ifndef _LP64
2415   } else if (left->is_single_fpu()) {
2416     assert(dest->is_single_fpu(),  "fpu stack allocation required");
2417 
2418     if (right->is_single_fpu()) {
2419       arith_fpu_implementation(code, left->fpu_regnr(), right->fpu_regnr(), dest->fpu_regnr(), pop_fpu_stack);
2420 
2421     } else {
2422       assert(left->fpu_regnr() == 0, "left must be on TOS");
2423       assert(dest->fpu_regnr() == 0, "dest must be on TOS");
2424 
2425       Address raddr;
2426       if (right->is_single_stack()) {
2427         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2428       } else if (right->is_constant()) {
2429         address const_addr = float_constant(right->as_jfloat());
2430         assert(const_addr != NULL, "incorrect float/double constant maintenance");
2431         // hack for now
2432         raddr = __ as_Address(InternalAddress(const_addr));
2433       } else {
2434         ShouldNotReachHere();
2435       }
2436 
2437       switch (code) {
2438         case lir_add: __ fadd_s(raddr); break;
2439         case lir_sub: __ fsub_s(raddr); break;
2440         case lir_mul: __ fmul_s(raddr); break;
2441         case lir_div: __ fdiv_s(raddr); break;
2442         default:      ShouldNotReachHere();
2443       }
2444     }
2445 
2446   } else if (left->is_double_fpu()) {
2447     assert(dest->is_double_fpu(),  "fpu stack allocation required");
2448 
2449     if (code == lir_mul || code == lir_div) {
2450       // Double values require special handling for strictfp mul/div on x86
2451       __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias1()));
2452       __ fmulp(left->fpu_regnrLo() + 1);
2453     }
2454 
2455     if (right->is_double_fpu()) {
2456       arith_fpu_implementation(code, left->fpu_regnrLo(), right->fpu_regnrLo(), dest->fpu_regnrLo(), pop_fpu_stack);
2457 
2458     } else {
2459       assert(left->fpu_regnrLo() == 0, "left must be on TOS");
2460       assert(dest->fpu_regnrLo() == 0, "dest must be on TOS");
2461 
2462       Address raddr;
2463       if (right->is_double_stack()) {
2464         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2465       } else if (right->is_constant()) {
2466         // hack for now
2467         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2468       } else {
2469         ShouldNotReachHere();
2470       }
2471 
2472       switch (code) {
2473         case lir_add: __ fadd_d(raddr); break;
2474         case lir_sub: __ fsub_d(raddr); break;
2475         case lir_mul: __ fmul_d(raddr); break;
2476         case lir_div: __ fdiv_d(raddr); break;
2477         default: ShouldNotReachHere();
2478       }
2479     }
2480 
2481     if (code == lir_mul || code == lir_div) {
2482       // Double values require special handling for strictfp mul/div on x86
2483       __ fld_x(ExternalAddress(StubRoutines::x86::addr_fpu_subnormal_bias2()));
2484       __ fmulp(dest->fpu_regnrLo() + 1);
2485     }
2486 #endif // !_LP64
2487 
2488   } else if (left->is_single_stack() || left->is_address()) {
2489     assert(left == dest, "left and dest must be equal");
2490 
2491     Address laddr;
2492     if (left->is_single_stack()) {
2493       laddr = frame_map()->address_for_slot(left->single_stack_ix());
2494     } else if (left->is_address()) {
2495       laddr = as_Address(left->as_address_ptr());
2496     } else {
2497       ShouldNotReachHere();
2498     }
2499 
2500     if (right->is_single_cpu()) {
2501       Register rreg = right->as_register();
2502       switch (code) {
2503         case lir_add: __ addl(laddr, rreg); break;
2504         case lir_sub: __ subl(laddr, rreg); break;
2505         default:      ShouldNotReachHere();
2506       }
2507     } else if (right->is_constant()) {
2508       jint c = right->as_constant_ptr()->as_jint();
2509       switch (code) {
2510         case lir_add: {
2511           __ incrementl(laddr, c);
2512           break;
2513         }
2514         case lir_sub: {
2515           __ decrementl(laddr, c);
2516           break;
2517         }
2518         default: ShouldNotReachHere();
2519       }
2520     } else {
2521       ShouldNotReachHere();
2522     }
2523 
2524   } else {
2525     ShouldNotReachHere();
2526   }
2527 }
2528 
2529 #ifndef _LP64
2530 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) {
2531   assert(pop_fpu_stack  || (left_index     == dest_index || right_index     == dest_index), "invalid LIR");
2532   assert(!pop_fpu_stack || (left_index - 1 == dest_index || right_index - 1 == dest_index), "invalid LIR");
2533   assert(left_index == 0 || right_index == 0, "either must be on top of stack");
2534 
2535   bool left_is_tos = (left_index == 0);
2536   bool dest_is_tos = (dest_index == 0);
2537   int non_tos_index = (left_is_tos ? right_index : left_index);
2538 
2539   switch (code) {
2540     case lir_add:
2541       if (pop_fpu_stack)       __ faddp(non_tos_index);
2542       else if (dest_is_tos)    __ fadd (non_tos_index);
2543       else                     __ fadda(non_tos_index);
2544       break;
2545 
2546     case lir_sub:
2547       if (left_is_tos) {
2548         if (pop_fpu_stack)     __ fsubrp(non_tos_index);
2549         else if (dest_is_tos)  __ fsub  (non_tos_index);
2550         else                   __ fsubra(non_tos_index);
2551       } else {
2552         if (pop_fpu_stack)     __ fsubp (non_tos_index);
2553         else if (dest_is_tos)  __ fsubr (non_tos_index);
2554         else                   __ fsuba (non_tos_index);
2555       }
2556       break;
2557 
2558     case lir_mul:
2559       if (pop_fpu_stack)       __ fmulp(non_tos_index);
2560       else if (dest_is_tos)    __ fmul (non_tos_index);
2561       else                     __ fmula(non_tos_index);
2562       break;
2563 
2564     case lir_div:
2565       if (left_is_tos) {
2566         if (pop_fpu_stack)     __ fdivrp(non_tos_index);
2567         else if (dest_is_tos)  __ fdiv  (non_tos_index);
2568         else                   __ fdivra(non_tos_index);
2569       } else {
2570         if (pop_fpu_stack)     __ fdivp (non_tos_index);
2571         else if (dest_is_tos)  __ fdivr (non_tos_index);
2572         else                   __ fdiva (non_tos_index);
2573       }
2574       break;
2575 
2576     case lir_rem:
2577       assert(left_is_tos && dest_is_tos && right_index == 1, "must be guaranteed by FPU stack allocation");
2578       __ fremr(noreg);
2579       break;
2580 
2581     default:
2582       ShouldNotReachHere();
2583   }
2584 }
2585 #endif // _LP64
2586 
2587 
2588 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2589   if (value->is_double_xmm()) {
2590     switch(code) {
2591       case lir_abs :
2592         {
2593 #ifdef _LP64
2594           if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
2595             assert(tmp->is_valid(), "need temporary");
2596             __ vpandn(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), value->as_xmm_double_reg(), 2);
2597           } else
2598 #endif
2599           {
2600             if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2601               __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2602             }
2603             assert(!tmp->is_valid(), "do not need temporary");
2604             __ andpd(dest->as_xmm_double_reg(),
2605                      ExternalAddress((address)double_signmask_pool),
2606                      rscratch1);
2607           }
2608         }
2609         break;
2610 
2611       case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2612       // all other intrinsics are not available in the SSE instruction set, so FPU is used
2613       default      : ShouldNotReachHere();
2614     }
2615 
2616 #ifndef _LP64
2617   } else if (value->is_double_fpu()) {
2618     assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2619     switch(code) {
2620       case lir_abs   : __ fabs() ; break;
2621       case lir_sqrt  : __ fsqrt(); break;
2622       default      : ShouldNotReachHere();
2623     }
2624 #endif // !_LP64
2625   } else {
2626     Unimplemented();
2627   }
2628 }
2629 
2630 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2631   // assert(left->destroys_register(), "check");
2632   if (left->is_single_cpu()) {
2633     Register reg = left->as_register();
2634     if (right->is_constant()) {
2635       int val = right->as_constant_ptr()->as_jint();
2636       switch (code) {
2637         case lir_logic_and: __ andl (reg, val); break;
2638         case lir_logic_or:  __ orl  (reg, val); break;
2639         case lir_logic_xor: __ xorl (reg, val); break;
2640         default: ShouldNotReachHere();
2641       }
2642     } else if (right->is_stack()) {
2643       // added support for stack operands
2644       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2645       switch (code) {
2646         case lir_logic_and: __ andl (reg, raddr); break;
2647         case lir_logic_or:  __ orl  (reg, raddr); break;
2648         case lir_logic_xor: __ xorl (reg, raddr); break;
2649         default: ShouldNotReachHere();
2650       }
2651     } else {
2652       Register rright = right->as_register();
2653       switch (code) {
2654         case lir_logic_and: __ andptr (reg, rright); break;
2655         case lir_logic_or : __ orptr  (reg, rright); break;
2656         case lir_logic_xor: __ xorptr (reg, rright); break;
2657         default: ShouldNotReachHere();
2658       }
2659     }
2660     move_regs(reg, dst->as_register());
2661   } else {
2662     Register l_lo = left->as_register_lo();
2663     Register l_hi = left->as_register_hi();
2664     if (right->is_constant()) {
2665 #ifdef _LP64
2666       __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2667       switch (code) {
2668         case lir_logic_and:
2669           __ andq(l_lo, rscratch1);
2670           break;
2671         case lir_logic_or:
2672           __ orq(l_lo, rscratch1);
2673           break;
2674         case lir_logic_xor:
2675           __ xorq(l_lo, rscratch1);
2676           break;
2677         default: ShouldNotReachHere();
2678       }
2679 #else
2680       int r_lo = right->as_constant_ptr()->as_jint_lo();
2681       int r_hi = right->as_constant_ptr()->as_jint_hi();
2682       switch (code) {
2683         case lir_logic_and:
2684           __ andl(l_lo, r_lo);
2685           __ andl(l_hi, r_hi);
2686           break;
2687         case lir_logic_or:
2688           __ orl(l_lo, r_lo);
2689           __ orl(l_hi, r_hi);
2690           break;
2691         case lir_logic_xor:
2692           __ xorl(l_lo, r_lo);
2693           __ xorl(l_hi, r_hi);
2694           break;
2695         default: ShouldNotReachHere();
2696       }
2697 #endif // _LP64
2698     } else {
2699 #ifdef _LP64
2700       Register r_lo;
2701       if (is_reference_type(right->type())) {
2702         r_lo = right->as_register();
2703       } else {
2704         r_lo = right->as_register_lo();
2705       }
2706 #else
2707       Register r_lo = right->as_register_lo();
2708       Register r_hi = right->as_register_hi();
2709       assert(l_lo != r_hi, "overwriting registers");
2710 #endif
2711       switch (code) {
2712         case lir_logic_and:
2713           __ andptr(l_lo, r_lo);
2714           NOT_LP64(__ andptr(l_hi, r_hi);)
2715           break;
2716         case lir_logic_or:
2717           __ orptr(l_lo, r_lo);
2718           NOT_LP64(__ orptr(l_hi, r_hi);)
2719           break;
2720         case lir_logic_xor:
2721           __ xorptr(l_lo, r_lo);
2722           NOT_LP64(__ xorptr(l_hi, r_hi);)
2723           break;
2724         default: ShouldNotReachHere();
2725       }
2726     }
2727 
2728     Register dst_lo = dst->as_register_lo();
2729     Register dst_hi = dst->as_register_hi();
2730 
2731 #ifdef _LP64
2732     move_regs(l_lo, dst_lo);
2733 #else
2734     if (dst_lo == l_hi) {
2735       assert(dst_hi != l_lo, "overwriting registers");
2736       move_regs(l_hi, dst_hi);
2737       move_regs(l_lo, dst_lo);
2738     } else {
2739       assert(dst_lo != l_hi, "overwriting registers");
2740       move_regs(l_lo, dst_lo);
2741       move_regs(l_hi, dst_hi);
2742     }
2743 #endif // _LP64
2744   }
2745 }
2746 
2747 
2748 // we assume that rax, and rdx can be overwritten
2749 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2750 
2751   assert(left->is_single_cpu(),   "left must be register");
2752   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2753   assert(result->is_single_cpu(), "result must be register");
2754 
2755   //  assert(left->destroys_register(), "check");
2756   //  assert(right->destroys_register(), "check");
2757 
2758   Register lreg = left->as_register();
2759   Register dreg = result->as_register();
2760 
2761   if (right->is_constant()) {
2762     jint divisor = right->as_constant_ptr()->as_jint();
2763     assert(divisor > 0 && is_power_of_2(divisor), "must be");
2764     if (code == lir_idiv) {
2765       assert(lreg == rax, "must be rax,");
2766       assert(temp->as_register() == rdx, "tmp register must be rdx");
2767       __ cdql(); // sign extend into rdx:rax
2768       if (divisor == 2) {
2769         __ subl(lreg, rdx);
2770       } else {
2771         __ andl(rdx, divisor - 1);
2772         __ addl(lreg, rdx);
2773       }
2774       __ sarl(lreg, log2i_exact(divisor));
2775       move_regs(lreg, dreg);
2776     } else if (code == lir_irem) {
2777       Label done;
2778       __ mov(dreg, lreg);
2779       __ andl(dreg, 0x80000000 | (divisor - 1));
2780       __ jcc(Assembler::positive, done);
2781       __ decrement(dreg);
2782       __ orl(dreg, ~(divisor - 1));
2783       __ increment(dreg);
2784       __ bind(done);
2785     } else {
2786       ShouldNotReachHere();
2787     }
2788   } else {
2789     Register rreg = right->as_register();
2790     assert(lreg == rax, "left register must be rax,");
2791     assert(rreg != rdx, "right register must not be rdx");
2792     assert(temp->as_register() == rdx, "tmp register must be rdx");
2793 
2794     move_regs(lreg, rax);
2795 
2796     int idivl_offset = __ corrected_idivl(rreg);
2797     if (ImplicitDiv0Checks) {
2798       add_debug_info_for_div0(idivl_offset, info);
2799     }
2800     if (code == lir_irem) {
2801       move_regs(rdx, dreg); // result is in rdx
2802     } else {
2803       move_regs(rax, dreg);
2804     }
2805   }
2806 }
2807 
2808 
2809 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2810   if (opr1->is_single_cpu()) {
2811     Register reg1 = opr1->as_register();
2812     if (opr2->is_single_cpu()) {
2813       // cpu register - cpu register
2814       if (is_reference_type(opr1->type())) {
2815         __ cmpoop(reg1, opr2->as_register());
2816       } else {
2817         assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2818         __ cmpl(reg1, opr2->as_register());
2819       }
2820     } else if (opr2->is_stack()) {
2821       // cpu register - stack
2822       if (is_reference_type(opr1->type())) {
2823         __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2824       } else {
2825         __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2826       }
2827     } else if (opr2->is_constant()) {
2828       // cpu register - constant
2829       LIR_Const* c = opr2->as_constant_ptr();
2830       if (c->type() == T_INT) {
2831         __ cmpl(reg1, c->as_jint());
2832       } else if (c->type() == T_METADATA) {
2833         // All we need for now is a comparison with NULL for equality.
2834         assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "oops");
2835         Metadata* m = c->as_metadata();
2836         if (m == NULL) {
2837           __ cmpptr(reg1, NULL_WORD);
2838         } else {
2839           ShouldNotReachHere();
2840         }
2841       } else if (is_reference_type(c->type())) {
2842         // In 64bit oops are single register
2843         jobject o = c->as_jobject();
2844         if (o == NULL) {
2845           __ cmpptr(reg1, NULL_WORD);
2846         } else {
2847           __ cmpoop(reg1, o, rscratch1);
2848         }
2849       } else {
2850         fatal("unexpected type: %s", basictype_to_str(c->type()));
2851       }
2852       // cpu register - address
2853     } else if (opr2->is_address()) {
2854       if (op->info() != NULL) {
2855         add_debug_info_for_null_check_here(op->info());
2856       }
2857       __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2858     } else {
2859       ShouldNotReachHere();
2860     }
2861 
2862   } else if(opr1->is_double_cpu()) {
2863     Register xlo = opr1->as_register_lo();
2864     Register xhi = opr1->as_register_hi();
2865     if (opr2->is_double_cpu()) {
2866 #ifdef _LP64
2867       __ cmpptr(xlo, opr2->as_register_lo());
2868 #else
2869       // cpu register - cpu register
2870       Register ylo = opr2->as_register_lo();
2871       Register yhi = opr2->as_register_hi();
2872       __ subl(xlo, ylo);
2873       __ sbbl(xhi, yhi);
2874       if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
2875         __ orl(xhi, xlo);
2876       }
2877 #endif // _LP64
2878     } else if (opr2->is_constant()) {
2879       // cpu register - constant 0
2880       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2881 #ifdef _LP64
2882       __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2883 #else
2884       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles equals case");
2885       __ orl(xhi, xlo);
2886 #endif // _LP64
2887     } else {
2888       ShouldNotReachHere();
2889     }
2890 
2891   } else if (opr1->is_single_xmm()) {
2892     XMMRegister reg1 = opr1->as_xmm_float_reg();
2893     if (opr2->is_single_xmm()) {
2894       // xmm register - xmm register
2895       __ ucomiss(reg1, opr2->as_xmm_float_reg());
2896     } else if (opr2->is_stack()) {
2897       // xmm register - stack
2898       __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2899     } else if (opr2->is_constant()) {
2900       // xmm register - constant
2901       __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2902     } else if (opr2->is_address()) {
2903       // xmm register - address
2904       if (op->info() != NULL) {
2905         add_debug_info_for_null_check_here(op->info());
2906       }
2907       __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2908     } else {
2909       ShouldNotReachHere();
2910     }
2911 
2912   } else if (opr1->is_double_xmm()) {
2913     XMMRegister reg1 = opr1->as_xmm_double_reg();
2914     if (opr2->is_double_xmm()) {
2915       // xmm register - xmm register
2916       __ ucomisd(reg1, opr2->as_xmm_double_reg());
2917     } else if (opr2->is_stack()) {
2918       // xmm register - stack
2919       __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2920     } else if (opr2->is_constant()) {
2921       // xmm register - constant
2922       __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2923     } else if (opr2->is_address()) {
2924       // xmm register - address
2925       if (op->info() != NULL) {
2926         add_debug_info_for_null_check_here(op->info());
2927       }
2928       __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2929     } else {
2930       ShouldNotReachHere();
2931     }
2932 
2933 #ifndef _LP64
2934   } else if(opr1->is_single_fpu() || opr1->is_double_fpu()) {
2935     assert(opr1->is_fpu_register() && opr1->fpu() == 0, "currently left-hand side must be on TOS (relax this restriction)");
2936     assert(opr2->is_fpu_register(), "both must be registers");
2937     __ fcmp(noreg, opr2->fpu(), op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2938 #endif // LP64
2939 
2940   } else if (opr1->is_address() && opr2->is_constant()) {
2941     LIR_Const* c = opr2->as_constant_ptr();
2942 #ifdef _LP64
2943     if (is_reference_type(c->type())) {
2944       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2945       __ movoop(rscratch1, c->as_jobject());
2946     }
2947 #endif // LP64
2948     if (op->info() != NULL) {
2949       add_debug_info_for_null_check_here(op->info());
2950     }
2951     // special case: address - constant
2952     LIR_Address* addr = opr1->as_address_ptr();
2953     if (c->type() == T_INT) {
2954       __ cmpl(as_Address(addr), c->as_jint());
2955     } else if (is_reference_type(c->type())) {
2956 #ifdef _LP64
2957       // %%% Make this explode if addr isn't reachable until we figure out a
2958       // better strategy by giving noreg as the temp for as_Address
2959       __ cmpoop(rscratch1, as_Address(addr, noreg));
2960 #else
2961       __ cmpoop(as_Address(addr), c->as_jobject());
2962 #endif // _LP64
2963     } else {
2964       ShouldNotReachHere();
2965     }
2966 
2967   } else {
2968     ShouldNotReachHere();
2969   }
2970 }
2971 
2972 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2973   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2974     if (left->is_single_xmm()) {
2975       assert(right->is_single_xmm(), "must match");
2976       __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2977     } else if (left->is_double_xmm()) {
2978       assert(right->is_double_xmm(), "must match");
2979       __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2980 
2981     } else {
2982 #ifdef _LP64
2983       ShouldNotReachHere();
2984 #else
2985       assert(left->is_single_fpu() || left->is_double_fpu(), "must be");
2986       assert(right->is_single_fpu() || right->is_double_fpu(), "must match");
2987 
2988       assert(left->fpu() == 0, "left must be on TOS");
2989       __ fcmp2int(dst->as_register(), code == lir_ucmp_fd2i, right->fpu(),
2990                   op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2991 #endif // LP64
2992     }
2993   } else {
2994     assert(code == lir_cmp_l2i, "check");
2995 #ifdef _LP64
2996     Label done;
2997     Register dest = dst->as_register();
2998     __ cmpptr(left->as_register_lo(), right->as_register_lo());
2999     __ movl(dest, -1);
3000     __ jccb(Assembler::less, done);
3001     __ set_byte_if_not_zero(dest);
3002     __ movzbl(dest, dest);
3003     __ bind(done);
3004 #else
3005     __ lcmp2int(left->as_register_hi(),
3006                 left->as_register_lo(),
3007                 right->as_register_hi(),
3008                 right->as_register_lo());
3009     move_regs(left->as_register_hi(), dst->as_register());
3010 #endif // _LP64
3011   }
3012 }
3013 
3014 
3015 void LIR_Assembler::align_call(LIR_Code code) {
3016   // make sure that the displacement word of the call ends up word aligned
3017   int offset = __ offset();
3018   switch (code) {
3019   case lir_static_call:
3020   case lir_optvirtual_call:
3021   case lir_dynamic_call:
3022     offset += NativeCall::displacement_offset;
3023     break;
3024   case lir_icvirtual_call:
3025     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size;
3026     break;
3027   default: ShouldNotReachHere();
3028   }
3029   __ align(BytesPerWord, offset);
3030 }
3031 
3032 
3033 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
3034   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
3035          "must be aligned");
3036   __ call(AddressLiteral(op->addr(), rtype));
3037   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
3038   __ post_call_nop();
3039 }
3040 
3041 
3042 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
3043   __ ic_call(op->addr());
3044   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
3045   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
3046          "must be aligned");
3047   __ post_call_nop();
3048 }
3049 
3050 
3051 void LIR_Assembler::emit_static_call_stub() {
3052   address call_pc = __ pc();
3053   address stub = __ start_a_stub(call_stub_size());
3054   if (stub == NULL) {
3055     bailout("static call stub overflow");
3056     return;
3057   }
3058 
3059   int start = __ offset();
3060 
3061   // make sure that the displacement word of the call ends up word aligned
3062   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset);
3063   __ relocate(static_stub_Relocation::spec(call_pc));
3064   __ mov_metadata(rbx, (Metadata*)NULL);
3065   // must be set to -1 at code generation time
3066   assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
3067   // On 64bit this will die since it will take a movq & jmp, must be only a jmp
3068   __ jump(RuntimeAddress(__ pc()));
3069 
3070   assert(__ offset() - start <= call_stub_size(), "stub too big");
3071   __ end_a_stub();
3072 }
3073 
3074 
3075 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
3076   assert(exceptionOop->as_register() == rax, "must match");
3077   assert(exceptionPC->as_register() == rdx, "must match");
3078 
3079   // exception object is not added to oop map by LinearScan
3080   // (LinearScan assumes that no oops are in fixed registers)
3081   info->add_register_oop(exceptionOop);
3082   Runtime1::StubID unwind_id;
3083 
3084   // get current pc information
3085   // pc is only needed if the method has an exception handler, the unwind code does not need it.
3086   int pc_for_athrow_offset = __ offset();
3087   InternalAddress pc_for_athrow(__ pc());
3088   __ lea(exceptionPC->as_register(), pc_for_athrow);
3089   add_call_info(pc_for_athrow_offset, info); // for exception handler
3090 
3091   __ verify_not_null_oop(rax);
3092   // search an exception handler (rax: exception oop, rdx: throwing pc)
3093   if (compilation()->has_fpu_code()) {
3094     unwind_id = Runtime1::handle_exception_id;
3095   } else {
3096     unwind_id = Runtime1::handle_exception_nofpu_id;
3097   }
3098   __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
3099 
3100   // enough room for two byte trap
3101   __ nop();
3102 }
3103 
3104 
3105 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
3106   assert(exceptionOop->as_register() == rax, "must match");
3107 
3108   __ jmp(_unwind_handler_entry);
3109 }
3110 
3111 
3112 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
3113 
3114   // optimized version for linear scan:
3115   // * count must be already in ECX (guaranteed by LinearScan)
3116   // * left and dest must be equal
3117   // * tmp must be unused
3118   assert(count->as_register() == SHIFT_count, "count must be in ECX");
3119   assert(left == dest, "left and dest must be equal");
3120   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
3121 
3122   if (left->is_single_cpu()) {
3123     Register value = left->as_register();
3124     assert(value != SHIFT_count, "left cannot be ECX");
3125 
3126     switch (code) {
3127       case lir_shl:  __ shll(value); break;
3128       case lir_shr:  __ sarl(value); break;
3129       case lir_ushr: __ shrl(value); break;
3130       default: ShouldNotReachHere();
3131     }
3132   } else if (left->is_double_cpu()) {
3133     Register lo = left->as_register_lo();
3134     Register hi = left->as_register_hi();
3135     assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
3136 #ifdef _LP64
3137     switch (code) {
3138       case lir_shl:  __ shlptr(lo);        break;
3139       case lir_shr:  __ sarptr(lo);        break;
3140       case lir_ushr: __ shrptr(lo);        break;
3141       default: ShouldNotReachHere();
3142     }
3143 #else
3144 
3145     switch (code) {
3146       case lir_shl:  __ lshl(hi, lo);        break;
3147       case lir_shr:  __ lshr(hi, lo, true);  break;
3148       case lir_ushr: __ lshr(hi, lo, false); break;
3149       default: ShouldNotReachHere();
3150     }
3151 #endif // LP64
3152   } else {
3153     ShouldNotReachHere();
3154   }
3155 }
3156 
3157 
3158 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
3159   if (dest->is_single_cpu()) {
3160     // first move left into dest so that left is not destroyed by the shift
3161     Register value = dest->as_register();
3162     count = count & 0x1F; // Java spec
3163 
3164     move_regs(left->as_register(), value);
3165     switch (code) {
3166       case lir_shl:  __ shll(value, count); break;
3167       case lir_shr:  __ sarl(value, count); break;
3168       case lir_ushr: __ shrl(value, count); break;
3169       default: ShouldNotReachHere();
3170     }
3171   } else if (dest->is_double_cpu()) {
3172 #ifndef _LP64
3173     Unimplemented();
3174 #else
3175     // first move left into dest so that left is not destroyed by the shift
3176     Register value = dest->as_register_lo();
3177     count = count & 0x1F; // Java spec
3178 
3179     move_regs(left->as_register_lo(), value);
3180     switch (code) {
3181       case lir_shl:  __ shlptr(value, count); break;
3182       case lir_shr:  __ sarptr(value, count); break;
3183       case lir_ushr: __ shrptr(value, count); break;
3184       default: ShouldNotReachHere();
3185     }
3186 #endif // _LP64
3187   } else {
3188     ShouldNotReachHere();
3189   }
3190 }
3191 
3192 
3193 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
3194   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3195   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3196   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3197   __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
3198 }
3199 
3200 
3201 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
3202   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3203   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3204   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3205   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3206 }
3207 
3208 
3209 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
3210   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3211   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3212   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3213   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
3214 }
3215 
3216 
3217 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
3218   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3219   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3220   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3221   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
3222 }
3223 
3224 
3225 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
3226   if (null_check) {
3227     __ testptr(obj, obj);
3228     __ jcc(Assembler::zero, *slow_path->entry());
3229   }
3230   if (UseArrayMarkWordCheck) {
3231     if (is_dest) {
3232       __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
3233     } else {
3234       __ test_flattened_array_oop(obj, tmp, *slow_path->entry());
3235     }
3236   } else {
3237     Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3238     __ load_klass(tmp, obj, tmp_load_klass);
3239     __ movl(tmp, Address(tmp, Klass::layout_helper_offset()));
3240     if (is_dest) {
3241       // Take the slow path if it's a null_free destination array, in case the source array contains NULLs.
3242       __ testl(tmp, Klass::_lh_null_free_array_bit_inplace);
3243     } else {
3244       __ testl(tmp, Klass::_lh_array_tag_flat_value_bit_inplace);
3245     }
3246     __ jcc(Assembler::notZero, *slow_path->entry());
3247   }
3248 }
3249 
3250 
3251 // This code replaces a call to arraycopy; no exception may
3252 // be thrown in this code, they must be thrown in the System.arraycopy
3253 // activation frame; we could save some checks if this would not be the case
3254 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3255   ciArrayKlass* default_type = op->expected_type();
3256   Register src = op->src()->as_register();
3257   Register dst = op->dst()->as_register();
3258   Register src_pos = op->src_pos()->as_register();
3259   Register dst_pos = op->dst_pos()->as_register();
3260   Register length  = op->length()->as_register();
3261   Register tmp = op->tmp()->as_register();
3262   Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3263 
3264   CodeStub* stub = op->stub();
3265   int flags = op->flags();
3266   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
3267   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
3268 
3269   if (flags & LIR_OpArrayCopy::always_slow_path) {
3270     __ jmp(*stub->entry());
3271     __ bind(*stub->continuation());
3272     return;
3273   }
3274 
3275   // if we don't know anything, just go through the generic arraycopy
3276   if (default_type == NULL) {
3277     // save outgoing arguments on stack in case call to System.arraycopy is needed
3278     // HACK ALERT. This code used to push the parameters in a hardwired fashion
3279     // for interpreter calling conventions. Now we have to do it in new style conventions.
3280     // For the moment until C1 gets the new register allocator I just force all the
3281     // args to the right place (except the register args) and then on the back side
3282     // reload the register args properly if we go slow path. Yuck
3283 
3284     // These are proper for the calling convention
3285     store_parameter(length, 2);
3286     store_parameter(dst_pos, 1);
3287     store_parameter(dst, 0);
3288 
3289     // these are just temporary placements until we need to reload
3290     store_parameter(src_pos, 3);
3291     store_parameter(src, 4);
3292     NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3293 
3294     address copyfunc_addr = StubRoutines::generic_arraycopy();
3295     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
3296 
3297     // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
3298 #ifdef _LP64
3299     // The arguments are in java calling convention so we can trivially shift them to C
3300     // convention
3301     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
3302     __ mov(c_rarg0, j_rarg0);
3303     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
3304     __ mov(c_rarg1, j_rarg1);
3305     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
3306     __ mov(c_rarg2, j_rarg2);
3307     assert_different_registers(c_rarg3, j_rarg4);
3308     __ mov(c_rarg3, j_rarg3);
3309 #ifdef _WIN64
3310     // Allocate abi space for args but be sure to keep stack aligned
3311     __ subptr(rsp, 6*wordSize);
3312     store_parameter(j_rarg4, 4);
3313 #ifndef PRODUCT
3314     if (PrintC1Statistics) {
3315       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
3316     }
3317 #endif
3318     __ call(RuntimeAddress(copyfunc_addr));
3319     __ addptr(rsp, 6*wordSize);
3320 #else
3321     __ mov(c_rarg4, j_rarg4);
3322 #ifndef PRODUCT
3323     if (PrintC1Statistics) {
3324       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
3325     }
3326 #endif
3327     __ call(RuntimeAddress(copyfunc_addr));
3328 #endif // _WIN64
3329 #else
3330     __ push(length);
3331     __ push(dst_pos);
3332     __ push(dst);
3333     __ push(src_pos);
3334     __ push(src);
3335 
3336 #ifndef PRODUCT
3337     if (PrintC1Statistics) {
3338       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt), rscratch1);
3339     }
3340 #endif
3341     __ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack
3342 
3343 #endif // _LP64
3344 
3345     __ cmpl(rax, 0);
3346     __ jcc(Assembler::equal, *stub->continuation());
3347 
3348     __ mov(tmp, rax);
3349     __ xorl(tmp, -1);
3350 
3351     // Reload values from the stack so they are where the stub
3352     // expects them.
3353     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3354     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3355     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3356     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3357     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3358 
3359     __ subl(length, tmp);
3360     __ addl(src_pos, tmp);
3361     __ addl(dst_pos, tmp);
3362     __ jmp(*stub->entry());
3363 
3364     __ bind(*stub->continuation());
3365     return;
3366   }
3367 
3368   // Handle inline type arrays
3369   if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
3370     arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
3371   }
3372   if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
3373     arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
3374   }
3375 
3376   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3377 
3378   int elem_size = type2aelembytes(basic_type);
3379   Address::ScaleFactor scale;
3380 
3381   switch (elem_size) {
3382     case 1 :
3383       scale = Address::times_1;
3384       break;
3385     case 2 :
3386       scale = Address::times_2;
3387       break;
3388     case 4 :
3389       scale = Address::times_4;
3390       break;
3391     case 8 :
3392       scale = Address::times_8;
3393       break;
3394     default:
3395       scale = Address::no_scale;
3396       ShouldNotReachHere();
3397   }
3398 
3399   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
3400   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
3401   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
3402   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
3403 
3404   // length and pos's are all sign extended at this point on 64bit
3405 
3406   // test for NULL
3407   if (flags & LIR_OpArrayCopy::src_null_check) {
3408     __ testptr(src, src);
3409     __ jcc(Assembler::zero, *stub->entry());
3410   }
3411   if (flags & LIR_OpArrayCopy::dst_null_check) {
3412     __ testptr(dst, dst);
3413     __ jcc(Assembler::zero, *stub->entry());
3414   }
3415 
3416   // If the compiler was not able to prove that exact type of the source or the destination
3417   // of the arraycopy is an array type, check at runtime if the source or the destination is
3418   // an instance type.
3419   if (flags & LIR_OpArrayCopy::type_check) {
3420     if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3421       __ load_klass(tmp, dst, tmp_load_klass);
3422       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3423       __ jcc(Assembler::greaterEqual, *stub->entry());
3424     }
3425 
3426     if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3427       __ load_klass(tmp, src, tmp_load_klass);
3428       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3429       __ jcc(Assembler::greaterEqual, *stub->entry());
3430     }
3431   }
3432 
3433   // check if negative
3434   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
3435     __ testl(src_pos, src_pos);
3436     __ jcc(Assembler::less, *stub->entry());
3437   }
3438   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
3439     __ testl(dst_pos, dst_pos);
3440     __ jcc(Assembler::less, *stub->entry());
3441   }
3442 
3443   if (flags & LIR_OpArrayCopy::src_range_check) {
3444     __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
3445     __ cmpl(tmp, src_length_addr);
3446     __ jcc(Assembler::above, *stub->entry());
3447   }
3448   if (flags & LIR_OpArrayCopy::dst_range_check) {
3449     __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3450     __ cmpl(tmp, dst_length_addr);
3451     __ jcc(Assembler::above, *stub->entry());
3452   }
3453 
3454   if (flags & LIR_OpArrayCopy::length_positive_check) {
3455     __ testl(length, length);
3456     __ jcc(Assembler::less, *stub->entry());
3457   }
3458 
3459 #ifdef _LP64
3460   __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3461   __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3462 #endif
3463 
3464   if (flags & LIR_OpArrayCopy::type_check) {
3465     // We don't know the array types are compatible
3466     if (basic_type != T_OBJECT) {
3467       // Simple test for basic type arrays
3468       if (UseCompressedClassPointers) {
3469         __ movl(tmp, src_klass_addr);
3470         __ cmpl(tmp, dst_klass_addr);
3471       } else {
3472         __ movptr(tmp, src_klass_addr);
3473         __ cmpptr(tmp, dst_klass_addr);
3474       }
3475       __ jcc(Assembler::notEqual, *stub->entry());
3476     } else {
3477       // For object arrays, if src is a sub class of dst then we can
3478       // safely do the copy.
3479       Label cont, slow;
3480 
3481       __ push(src);
3482       __ push(dst);
3483 
3484       __ load_klass(src, src, tmp_load_klass);
3485       __ load_klass(dst, dst, tmp_load_klass);
3486 
3487       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
3488 
3489       __ push(src);
3490       __ push(dst);
3491       __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
3492       __ pop(dst);
3493       __ pop(src);
3494 
3495       __ cmpl(src, 0);
3496       __ jcc(Assembler::notEqual, cont);
3497 
3498       __ bind(slow);
3499       __ pop(dst);
3500       __ pop(src);
3501 
3502       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
3503       if (copyfunc_addr != NULL) { // use stub if available
3504         // src is not a sub class of dst so we have to do a
3505         // per-element check.
3506 
3507         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
3508         if ((flags & mask) != mask) {
3509           // Check that at least both of them object arrays.
3510           assert(flags & mask, "one of the two should be known to be an object array");
3511 
3512           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3513             __ load_klass(tmp, src, tmp_load_klass);
3514           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3515             __ load_klass(tmp, dst, tmp_load_klass);
3516           }
3517           int lh_offset = in_bytes(Klass::layout_helper_offset());
3518           Address klass_lh_addr(tmp, lh_offset);
3519           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
3520           __ cmpl(klass_lh_addr, objArray_lh);
3521           __ jcc(Assembler::notEqual, *stub->entry());
3522         }
3523 
3524        // Spill because stubs can use any register they like and it's
3525        // easier to restore just those that we care about.
3526        store_parameter(dst, 0);
3527        store_parameter(dst_pos, 1);
3528        store_parameter(length, 2);
3529        store_parameter(src_pos, 3);
3530        store_parameter(src, 4);
3531 
3532 #ifndef _LP64
3533         __ movptr(tmp, dst_klass_addr);
3534         __ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset()));
3535         __ push(tmp);
3536         __ movl(tmp, Address(tmp, Klass::super_check_offset_offset()));
3537         __ push(tmp);
3538         __ push(length);
3539         __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3540         __ push(tmp);
3541         __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3542         __ push(tmp);
3543 
3544         __ call_VM_leaf(copyfunc_addr, 5);
3545 #else
3546         __ movl2ptr(length, length); //higher 32bits must be null
3547 
3548         __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3549         assert_different_registers(c_rarg0, dst, dst_pos, length);
3550         __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3551         assert_different_registers(c_rarg1, dst, length);
3552 
3553         __ mov(c_rarg2, length);
3554         assert_different_registers(c_rarg2, dst);
3555 
3556 #ifdef _WIN64
3557         // Allocate abi space for args but be sure to keep stack aligned
3558         __ subptr(rsp, 6*wordSize);
3559         __ load_klass(c_rarg3, dst, tmp_load_klass);
3560         __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
3561         store_parameter(c_rarg3, 4);
3562         __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
3563         __ call(RuntimeAddress(copyfunc_addr));
3564         __ addptr(rsp, 6*wordSize);
3565 #else
3566         __ load_klass(c_rarg4, dst, tmp_load_klass);
3567         __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
3568         __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
3569         __ call(RuntimeAddress(copyfunc_addr));
3570 #endif
3571 
3572 #endif
3573 
3574 #ifndef PRODUCT
3575         if (PrintC1Statistics) {
3576           Label failed;
3577           __ testl(rax, rax);
3578           __ jcc(Assembler::notZero, failed);
3579           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt), rscratch1);
3580           __ bind(failed);
3581         }
3582 #endif
3583 
3584         __ testl(rax, rax);
3585         __ jcc(Assembler::zero, *stub->continuation());
3586 
3587 #ifndef PRODUCT
3588         if (PrintC1Statistics) {
3589           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt), rscratch1);
3590         }
3591 #endif
3592 
3593         __ mov(tmp, rax);
3594 
3595         __ xorl(tmp, -1);
3596 
3597         // Restore previously spilled arguments
3598         __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3599         __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3600         __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3601         __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3602         __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3603 
3604 
3605         __ subl(length, tmp);
3606         __ addl(src_pos, tmp);
3607         __ addl(dst_pos, tmp);
3608       }
3609 
3610       __ jmp(*stub->entry());
3611 
3612       __ bind(cont);
3613       __ pop(dst);
3614       __ pop(src);
3615     }
3616   }
3617 
3618 #ifdef ASSERT
3619   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
3620     // Sanity check the known type with the incoming class.  For the
3621     // primitive case the types must match exactly with src.klass and
3622     // dst.klass each exactly matching the default type.  For the
3623     // object array case, if no type check is needed then either the
3624     // dst type is exactly the expected type and the src type is a
3625     // subtype which we can't check or src is the same array as dst
3626     // but not necessarily exactly of type default_type.
3627     Label known_ok, halt;
3628     __ mov_metadata(tmp, default_type->constant_encoding());
3629 #ifdef _LP64
3630     if (UseCompressedClassPointers) {
3631       __ encode_klass_not_null(tmp, rscratch1);
3632     }
3633 #endif
3634 
3635     if (basic_type != T_OBJECT) {
3636 
3637       if (UseCompressedClassPointers)          __ cmpl(tmp, dst_klass_addr);
3638       else                   __ cmpptr(tmp, dst_klass_addr);
3639       __ jcc(Assembler::notEqual, halt);
3640       if (UseCompressedClassPointers)          __ cmpl(tmp, src_klass_addr);
3641       else                   __ cmpptr(tmp, src_klass_addr);
3642       __ jcc(Assembler::equal, known_ok);
3643     } else {
3644       if (UseCompressedClassPointers)          __ cmpl(tmp, dst_klass_addr);
3645       else                   __ cmpptr(tmp, dst_klass_addr);
3646       __ jcc(Assembler::equal, known_ok);
3647       __ cmpptr(src, dst);
3648       __ jcc(Assembler::equal, known_ok);
3649     }
3650     __ bind(halt);
3651     __ stop("incorrect type information in arraycopy");
3652     __ bind(known_ok);
3653   }
3654 #endif
3655 
3656 #ifndef PRODUCT
3657   if (PrintC1Statistics) {
3658     __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
3659   }
3660 #endif
3661 
3662 #ifdef _LP64
3663   assert_different_registers(c_rarg0, dst, dst_pos, length);
3664   __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3665   assert_different_registers(c_rarg1, length);
3666   __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3667   __ mov(c_rarg2, length);
3668 
3669 #else
3670   __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3671   store_parameter(tmp, 0);
3672   __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3673   store_parameter(tmp, 1);
3674   store_parameter(length, 2);
3675 #endif // _LP64
3676 
3677   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
3678   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
3679   const char *name;
3680   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
3681   __ call_VM_leaf(entry, 0);
3682 
3683   __ bind(*stub->continuation());
3684 }
3685 
3686 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3687   assert(op->crc()->is_single_cpu(),  "crc must be register");
3688   assert(op->val()->is_single_cpu(),  "byte value must be register");
3689   assert(op->result_opr()->is_single_cpu(), "result must be register");
3690   Register crc = op->crc()->as_register();
3691   Register val = op->val()->as_register();
3692   Register res = op->result_opr()->as_register();
3693 
3694   assert_different_registers(val, crc, res);
3695 
3696   __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
3697   __ notl(crc); // ~crc
3698   __ update_byte_crc32(crc, val, res);
3699   __ notl(crc); // ~crc
3700   __ mov(res, crc);
3701 }
3702 
3703 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
3704   Register obj = op->obj_opr()->as_register();  // may not be an oop
3705   Register hdr = op->hdr_opr()->as_register();
3706   Register lock = op->lock_opr()->as_register();
3707   if (UseHeavyMonitors) {
3708     if (op->info() != NULL) {
3709       add_debug_info_for_null_check_here(op->info());
3710       __ null_check(obj);
3711     }
3712     __ jmp(*op->stub()->entry());
3713   } else if (op->code() == lir_lock) {
3714     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3715     // add debug info for NullPointerException only if one is possible
3716     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
3717     if (op->info() != NULL) {
3718       add_debug_info_for_null_check(null_check_offset, op->info());
3719     }
3720     // done
3721   } else if (op->code() == lir_unlock) {
3722     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3723     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3724   } else {
3725     Unimplemented();
3726   }
3727   __ bind(*op->stub()->continuation());
3728 }
3729 
3730 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
3731   Register obj = op->obj()->as_pointer_register();
3732   Register result = op->result_opr()->as_pointer_register();
3733 
3734   CodeEmitInfo* info = op->info();
3735   if (info != NULL) {
3736     add_debug_info_for_null_check_here(info);
3737   }
3738 
3739 #ifdef _LP64
3740   if (UseCompressedClassPointers) {
3741     __ movl(result, Address(obj, oopDesc::klass_offset_in_bytes()));
3742     __ decode_klass_not_null(result, rscratch1);
3743   } else
3744 #endif
3745     __ movptr(result, Address(obj, oopDesc::klass_offset_in_bytes()));
3746 }
3747 
3748 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3749   ciMethod* method = op->profiled_method();
3750   int bci          = op->profiled_bci();
3751   ciMethod* callee = op->profiled_callee();
3752   Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3753 
3754   // Update counter for all call types
3755   ciMethodData* md = method->method_data_or_null();
3756   assert(md != NULL, "Sanity");
3757   ciProfileData* data = md->bci_to_data(bci);
3758   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
3759   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
3760   Register mdo  = op->mdo()->as_register();
3761   __ mov_metadata(mdo, md->constant_encoding());
3762   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
3763   // Perform additional virtual call profiling for invokevirtual and
3764   // invokeinterface bytecodes
3765   if (op->should_profile_receiver_type()) {
3766     assert(op->recv()->is_single_cpu(), "recv must be allocated");
3767     Register recv = op->recv()->as_register();
3768     assert_different_registers(mdo, recv);
3769     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3770     ciKlass* known_klass = op->known_holder();
3771     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
3772       // We know the type that will be seen at this call site; we can
3773       // statically update the MethodData* rather than needing to do
3774       // dynamic tests on the receiver type
3775 
3776       // NOTE: we should probably put a lock around this search to
3777       // avoid collisions by concurrent compilations
3778       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3779       uint i;
3780       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3781         ciKlass* receiver = vc_data->receiver(i);
3782         if (known_klass->equals(receiver)) {
3783           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3784           __ addptr(data_addr, DataLayout::counter_increment);
3785           return;
3786         }
3787       }
3788 
3789       // Receiver type not found in profile data; select an empty slot
3790 
3791       // Note that this is less efficient than it should be because it
3792       // always does a write to the receiver part of the
3793       // VirtualCallData rather than just the first time
3794       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3795         ciKlass* receiver = vc_data->receiver(i);
3796         if (receiver == NULL) {
3797           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3798           __ mov_metadata(recv_addr, known_klass->constant_encoding(), rscratch1);
3799           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3800           __ addptr(data_addr, DataLayout::counter_increment);
3801           return;
3802         }
3803       }
3804     } else {
3805       __ load_klass(recv, recv, tmp_load_klass);
3806       Label update_done;
3807       type_profile_helper(mdo, md, data, recv, &update_done);
3808       // Receiver did not match any saved receiver and there is no empty row for it.
3809       // Increment total counter to indicate polymorphic case.
3810       __ addptr(counter_addr, DataLayout::counter_increment);
3811 
3812       __ bind(update_done);
3813     }
3814   } else {
3815     // Static call
3816     __ addptr(counter_addr, DataLayout::counter_increment);
3817   }
3818 }
3819 
3820 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3821   Register obj = op->obj()->as_register();
3822   Register tmp = op->tmp()->as_pointer_register();
3823   Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
3824   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3825   ciKlass* exact_klass = op->exact_klass();
3826   intptr_t current_klass = op->current_klass();
3827   bool not_null = op->not_null();
3828   bool no_conflict = op->no_conflict();
3829 
3830   Label update, next, none;
3831 
3832   bool do_null = !not_null;
3833   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3834   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3835 
3836   assert(do_null || do_update, "why are we here?");
3837   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3838 
3839   __ verify_oop(obj);
3840 
3841   if (tmp != obj) {
3842     __ mov(tmp, obj);
3843   }
3844   if (do_null) {
3845     __ testptr(tmp, tmp);
3846     __ jccb(Assembler::notZero, update);
3847     if (!TypeEntries::was_null_seen(current_klass)) {
3848       __ orptr(mdo_addr, TypeEntries::null_seen);
3849     }
3850     if (do_update) {
3851 #ifndef ASSERT
3852       __ jmpb(next);
3853     }
3854 #else
3855       __ jmp(next);
3856     }
3857   } else {
3858     __ testptr(tmp, tmp);
3859     __ jcc(Assembler::notZero, update);
3860     __ stop("unexpected null obj");
3861 #endif
3862   }
3863 
3864   __ bind(update);
3865 
3866   if (do_update) {
3867 #ifdef ASSERT
3868     if (exact_klass != NULL) {
3869       Label ok;
3870       __ load_klass(tmp, tmp, tmp_load_klass);
3871       __ push(tmp);
3872       __ mov_metadata(tmp, exact_klass->constant_encoding());
3873       __ cmpptr(tmp, Address(rsp, 0));
3874       __ jcc(Assembler::equal, ok);
3875       __ stop("exact klass and actual klass differ");
3876       __ bind(ok);
3877       __ pop(tmp);
3878     }
3879 #endif
3880     if (!no_conflict) {
3881       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
3882         if (exact_klass != NULL) {
3883           __ mov_metadata(tmp, exact_klass->constant_encoding());
3884         } else {
3885           __ load_klass(tmp, tmp, tmp_load_klass);
3886         }
3887 
3888         __ xorptr(tmp, mdo_addr);
3889         __ testptr(tmp, TypeEntries::type_klass_mask);
3890         // klass seen before, nothing to do. The unknown bit may have been
3891         // set already but no need to check.
3892         __ jccb(Assembler::zero, next);
3893 
3894         __ testptr(tmp, TypeEntries::type_unknown);
3895         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3896 
3897         if (TypeEntries::is_type_none(current_klass)) {
3898           __ cmpptr(mdo_addr, 0);
3899           __ jccb(Assembler::equal, none);
3900           __ cmpptr(mdo_addr, TypeEntries::null_seen);
3901           __ jccb(Assembler::equal, none);
3902           // There is a chance that the checks above (re-reading profiling
3903           // data from memory) fail if another thread has just set the
3904           // profiling to this obj's klass
3905           __ xorptr(tmp, mdo_addr);
3906           __ testptr(tmp, TypeEntries::type_klass_mask);
3907           __ jccb(Assembler::zero, next);
3908         }
3909       } else {
3910         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3911                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3912 
3913         __ movptr(tmp, mdo_addr);
3914         __ testptr(tmp, TypeEntries::type_unknown);
3915         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3916       }
3917 
3918       // different than before. Cannot keep accurate profile.
3919       __ orptr(mdo_addr, TypeEntries::type_unknown);
3920 
3921       if (TypeEntries::is_type_none(current_klass)) {
3922         __ jmpb(next);
3923 
3924         __ bind(none);
3925         // first time here. Set profile type.
3926         __ movptr(mdo_addr, tmp);
3927       }
3928     } else {
3929       // There's a single possible klass at this profile point
3930       assert(exact_klass != NULL, "should be");
3931       if (TypeEntries::is_type_none(current_klass)) {
3932         __ mov_metadata(tmp, exact_klass->constant_encoding());
3933         __ xorptr(tmp, mdo_addr);
3934         __ testptr(tmp, TypeEntries::type_klass_mask);
3935 #ifdef ASSERT
3936         __ jcc(Assembler::zero, next);
3937 
3938         {
3939           Label ok;
3940           __ push(tmp);
3941           __ cmpptr(mdo_addr, 0);
3942           __ jcc(Assembler::equal, ok);
3943           __ cmpptr(mdo_addr, TypeEntries::null_seen);
3944           __ jcc(Assembler::equal, ok);
3945           // may have been set by another thread
3946           __ mov_metadata(tmp, exact_klass->constant_encoding());
3947           __ xorptr(tmp, mdo_addr);
3948           __ testptr(tmp, TypeEntries::type_mask);
3949           __ jcc(Assembler::zero, ok);
3950 
3951           __ stop("unexpected profiling mismatch");
3952           __ bind(ok);
3953           __ pop(tmp);
3954         }
3955 #else
3956         __ jccb(Assembler::zero, next);
3957 #endif
3958         // first time here. Set profile type.
3959         __ movptr(mdo_addr, tmp);
3960       } else {
3961         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3962                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3963 
3964         __ movptr(tmp, mdo_addr);
3965         __ testptr(tmp, TypeEntries::type_unknown);
3966         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3967 
3968         __ orptr(mdo_addr, TypeEntries::type_unknown);
3969       }
3970     }
3971 
3972     __ bind(next);
3973   }
3974 }
3975 
3976 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3977   Register obj = op->obj()->as_register();
3978   Register tmp = op->tmp()->as_pointer_register();
3979   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3980   bool not_null = op->not_null();
3981   int flag = op->flag();
3982 
3983   Label not_inline_type;
3984   if (!not_null) {
3985     __ testptr(obj, obj);
3986     __ jccb(Assembler::zero, not_inline_type);
3987   }
3988 
3989   __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3990 
3991   __ orb(mdo_addr, flag);
3992 
3993   __ bind(not_inline_type);
3994 }
3995 
3996 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3997   Unimplemented();
3998 }
3999 
4000 
4001 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
4002   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
4003 }
4004 
4005 
4006 void LIR_Assembler::align_backward_branch_target() {
4007   __ align(BytesPerWord);
4008 }
4009 
4010 
4011 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
4012   if (left->is_single_cpu()) {
4013     __ negl(left->as_register());
4014     move_regs(left->as_register(), dest->as_register());
4015 
4016   } else if (left->is_double_cpu()) {
4017     Register lo = left->as_register_lo();
4018 #ifdef _LP64
4019     Register dst = dest->as_register_lo();
4020     __ movptr(dst, lo);
4021     __ negptr(dst);
4022 #else
4023     Register hi = left->as_register_hi();
4024     __ lneg(hi, lo);
4025     if (dest->as_register_lo() == hi) {
4026       assert(dest->as_register_hi() != lo, "destroying register");
4027       move_regs(hi, dest->as_register_hi());
4028       move_regs(lo, dest->as_register_lo());
4029     } else {
4030       move_regs(lo, dest->as_register_lo());
4031       move_regs(hi, dest->as_register_hi());
4032     }
4033 #endif // _LP64
4034 
4035   } else if (dest->is_single_xmm()) {
4036 #ifdef _LP64
4037     if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
4038       assert(tmp->is_valid(), "need temporary");
4039       assert_different_registers(left->as_xmm_float_reg(), tmp->as_xmm_float_reg());
4040       __ vpxor(dest->as_xmm_float_reg(), tmp->as_xmm_float_reg(), left->as_xmm_float_reg(), 2);
4041     }
4042     else
4043 #endif
4044     {
4045       assert(!tmp->is_valid(), "do not need temporary");
4046       if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
4047         __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
4048       }
4049       __ xorps(dest->as_xmm_float_reg(),
4050                ExternalAddress((address)float_signflip_pool),
4051                rscratch1);
4052     }
4053   } else if (dest->is_double_xmm()) {
4054 #ifdef _LP64
4055     if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
4056       assert(tmp->is_valid(), "need temporary");
4057       assert_different_registers(left->as_xmm_double_reg(), tmp->as_xmm_double_reg());
4058       __ vpxor(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), left->as_xmm_double_reg(), 2);
4059     }
4060     else
4061 #endif
4062     {
4063       assert(!tmp->is_valid(), "do not need temporary");
4064       if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
4065         __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
4066       }
4067       __ xorpd(dest->as_xmm_double_reg(),
4068                ExternalAddress((address)double_signflip_pool),
4069                rscratch1);
4070     }
4071 #ifndef _LP64
4072   } else if (left->is_single_fpu() || left->is_double_fpu()) {
4073     assert(left->fpu() == 0, "arg must be on TOS");
4074     assert(dest->fpu() == 0, "dest must be TOS");
4075     __ fchs();
4076 #endif // !_LP64
4077 
4078   } else {
4079     ShouldNotReachHere();
4080   }
4081 }
4082 
4083 
4084 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
4085   assert(src->is_address(), "must be an address");
4086   assert(dest->is_register(), "must be a register");
4087 
4088   PatchingStub* patch = NULL;
4089   if (patch_code != lir_patch_none) {
4090     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
4091   }
4092 
4093   Register reg = dest->as_pointer_register();
4094   LIR_Address* addr = src->as_address_ptr();
4095   __ lea(reg, as_Address(addr));
4096 
4097   if (patch != NULL) {
4098     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
4099   }
4100 }
4101 
4102 
4103 
4104 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
4105   assert(!tmp->is_valid(), "don't need temporary");
4106   __ call(RuntimeAddress(dest));
4107   if (info != NULL) {
4108     add_call_info_here(info);
4109   }
4110   __ post_call_nop();
4111 }
4112 
4113 
4114 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
4115   assert(type == T_LONG, "only for volatile long fields");
4116 
4117   if (info != NULL) {
4118     add_debug_info_for_null_check_here(info);
4119   }
4120 
4121   if (src->is_double_xmm()) {
4122     if (dest->is_double_cpu()) {
4123 #ifdef _LP64
4124       __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
4125 #else
4126       __ movdl(dest->as_register_lo(), src->as_xmm_double_reg());
4127       __ psrlq(src->as_xmm_double_reg(), 32);
4128       __ movdl(dest->as_register_hi(), src->as_xmm_double_reg());
4129 #endif // _LP64
4130     } else if (dest->is_double_stack()) {
4131       __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
4132     } else if (dest->is_address()) {
4133       __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
4134     } else {
4135       ShouldNotReachHere();
4136     }
4137 
4138   } else if (dest->is_double_xmm()) {
4139     if (src->is_double_stack()) {
4140       __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
4141     } else if (src->is_address()) {
4142       __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
4143     } else {
4144       ShouldNotReachHere();
4145     }
4146 
4147 #ifndef _LP64
4148   } else if (src->is_double_fpu()) {
4149     assert(src->fpu_regnrLo() == 0, "must be TOS");
4150     if (dest->is_double_stack()) {
4151       __ fistp_d(frame_map()->address_for_slot(dest->double_stack_ix()));
4152     } else if (dest->is_address()) {
4153       __ fistp_d(as_Address(dest->as_address_ptr()));
4154     } else {
4155       ShouldNotReachHere();
4156     }
4157 
4158   } else if (dest->is_double_fpu()) {
4159     assert(dest->fpu_regnrLo() == 0, "must be TOS");
4160     if (src->is_double_stack()) {
4161       __ fild_d(frame_map()->address_for_slot(src->double_stack_ix()));
4162     } else if (src->is_address()) {
4163       __ fild_d(as_Address(src->as_address_ptr()));
4164     } else {
4165       ShouldNotReachHere();
4166     }
4167 #endif // !_LP64
4168 
4169   } else {
4170     ShouldNotReachHere();
4171   }
4172 }
4173 
4174 #ifdef ASSERT
4175 // emit run-time assertion
4176 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
4177   assert(op->code() == lir_assert, "must be");
4178 
4179   if (op->in_opr1()->is_valid()) {
4180     assert(op->in_opr2()->is_valid(), "both operands must be valid");
4181     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
4182   } else {
4183     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
4184     assert(op->condition() == lir_cond_always, "no other conditions allowed");
4185   }
4186 
4187   Label ok;
4188   if (op->condition() != lir_cond_always) {
4189     Assembler::Condition acond = Assembler::zero;
4190     switch (op->condition()) {
4191       case lir_cond_equal:        acond = Assembler::equal;       break;
4192       case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
4193       case lir_cond_less:         acond = Assembler::less;        break;
4194       case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
4195       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
4196       case lir_cond_greater:      acond = Assembler::greater;     break;
4197       case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
4198       case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
4199       default:                    ShouldNotReachHere();
4200     }
4201     __ jcc(acond, ok);
4202   }
4203   if (op->halt()) {
4204     const char* str = __ code_string(op->msg());
4205     __ stop(str);
4206   } else {
4207     breakpoint();
4208   }
4209   __ bind(ok);
4210 }
4211 #endif
4212 
4213 void LIR_Assembler::membar() {
4214   // QQQ sparc TSO uses this,
4215   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
4216 }
4217 
4218 void LIR_Assembler::membar_acquire() {
4219   // No x86 machines currently require load fences
4220 }
4221 
4222 void LIR_Assembler::membar_release() {
4223   // No x86 machines currently require store fences
4224 }
4225 
4226 void LIR_Assembler::membar_loadload() {
4227   // no-op
4228   //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
4229 }
4230 
4231 void LIR_Assembler::membar_storestore() {
4232   // no-op
4233   //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
4234 }
4235 
4236 void LIR_Assembler::membar_loadstore() {
4237   // no-op
4238   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
4239 }
4240 
4241 void LIR_Assembler::membar_storeload() {
4242   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
4243 }
4244 
4245 void LIR_Assembler::on_spin_wait() {
4246   __ pause ();
4247 }
4248 
4249 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
4250   assert(result_reg->is_register(), "check");
4251 #ifdef _LP64
4252   // __ get_thread(result_reg->as_register_lo());
4253   __ mov(result_reg->as_register(), r15_thread);
4254 #else
4255   __ get_thread(result_reg->as_register());
4256 #endif // _LP64
4257 }
4258 
4259 void LIR_Assembler::check_orig_pc() {
4260   __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
4261 }
4262 
4263 void LIR_Assembler::peephole(LIR_List*) {
4264   // do nothing for now
4265 }
4266 
4267 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
4268   assert(data == dest, "xchg/xadd uses only 2 operands");
4269 
4270   if (data->type() == T_INT) {
4271     if (code == lir_xadd) {
4272       __ lock();
4273       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
4274     } else {
4275       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
4276     }
4277   } else if (data->is_oop()) {
4278     assert (code == lir_xchg, "xadd for oops");
4279     Register obj = data->as_register();
4280 #ifdef _LP64
4281     if (UseCompressedOops) {
4282       __ encode_heap_oop(obj);
4283       __ xchgl(obj, as_Address(src->as_address_ptr()));
4284       __ decode_heap_oop(obj);
4285     } else {
4286       __ xchgptr(obj, as_Address(src->as_address_ptr()));
4287     }
4288 #else
4289     __ xchgl(obj, as_Address(src->as_address_ptr()));
4290 #endif
4291   } else if (data->type() == T_LONG) {
4292 #ifdef _LP64
4293     assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
4294     if (code == lir_xadd) {
4295       __ lock();
4296       __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
4297     } else {
4298       __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
4299     }
4300 #else
4301     ShouldNotReachHere();
4302 #endif
4303   } else {
4304     ShouldNotReachHere();
4305   }
4306 }
4307 
4308 #undef __