1 /*
   2  * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInstance.hpp"
  35 #include "ci/ciValueKlass.hpp"
  36 #include "gc/shared/barrierSet.hpp"
  37 #include "gc/shared/cardTableBarrierSet.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "nativeInst_x86.hpp"
  40 #include "oops/objArrayKlass.hpp"
  41 #include "runtime/frame.inline.hpp"
  42 #include "runtime/safepointMechanism.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "vmreg_x86.inline.hpp"
  45 
  46 
  47 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  48 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  49 // fast versions of NegF/NegD and AbsF/AbsD.
  50 
  51 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  52 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  53   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  54   // of 128-bits operands for SSE instructions.
  55   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  56   // Store the value to a 128-bits operand.
  57   operand[0] = lo;
  58   operand[1] = hi;
  59   return operand;
  60 }
  61 
  62 // Buffer for 128-bits masks used by SSE instructions.
  63 static jlong fp_signmask_pool[(4+1)*2]; // 4*128bits(data) + 128bits(alignment)
  64 
  65 // Static initialization during VM startup.
  66 static jlong *float_signmask_pool  = double_quadword(&fp_signmask_pool[1*2],         CONST64(0x7FFFFFFF7FFFFFFF),         CONST64(0x7FFFFFFF7FFFFFFF));
  67 static jlong *double_signmask_pool = double_quadword(&fp_signmask_pool[2*2],         CONST64(0x7FFFFFFFFFFFFFFF),         CONST64(0x7FFFFFFFFFFFFFFF));
  68 static jlong *float_signflip_pool  = double_quadword(&fp_signmask_pool[3*2], (jlong)UCONST64(0x8000000080000000), (jlong)UCONST64(0x8000000080000000));
  69 static jlong *double_signflip_pool = double_quadword(&fp_signmask_pool[4*2], (jlong)UCONST64(0x8000000000000000), (jlong)UCONST64(0x8000000000000000));
  70 
  71 
  72 NEEDS_CLEANUP // remove this definitions ?
  73 const Register IC_Klass    = rax;   // where the IC klass is cached
  74 const Register SYNC_header = rax;   // synchronization header
  75 const Register SHIFT_count = rcx;   // where count for shift operations must be
  76 
  77 #define __ _masm->
  78 
  79 
  80 static void select_different_registers(Register preserve,
  81                                        Register extra,
  82                                        Register &tmp1,
  83                                        Register &tmp2) {
  84   if (tmp1 == preserve) {
  85     assert_different_registers(tmp1, tmp2, extra);
  86     tmp1 = extra;
  87   } else if (tmp2 == preserve) {
  88     assert_different_registers(tmp1, tmp2, extra);
  89     tmp2 = extra;
  90   }
  91   assert_different_registers(preserve, tmp1, tmp2);
  92 }
  93 
  94 
  95 
  96 static void select_different_registers(Register preserve,
  97                                        Register extra,
  98                                        Register &tmp1,
  99                                        Register &tmp2,
 100                                        Register &tmp3) {
 101   if (tmp1 == preserve) {
 102     assert_different_registers(tmp1, tmp2, tmp3, extra);
 103     tmp1 = extra;
 104   } else if (tmp2 == preserve) {
 105     assert_different_registers(tmp1, tmp2, tmp3, extra);
 106     tmp2 = extra;
 107   } else if (tmp3 == preserve) {
 108     assert_different_registers(tmp1, tmp2, tmp3, extra);
 109     tmp3 = extra;
 110   }
 111   assert_different_registers(preserve, tmp1, tmp2, tmp3);
 112 }
 113 
 114 
 115 
 116 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
 117   if (opr->is_constant()) {
 118     LIR_Const* constant = opr->as_constant_ptr();
 119     switch (constant->type()) {
 120       case T_INT: {
 121         return true;
 122       }
 123 
 124       default:
 125         return false;
 126     }
 127   }
 128   return false;
 129 }
 130 
 131 
 132 LIR_Opr LIR_Assembler::receiverOpr() {
 133   return FrameMap::receiver_opr;
 134 }
 135 
 136 LIR_Opr LIR_Assembler::osrBufferPointer() {
 137   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 138 }
 139 
 140 //--------------fpu register translations-----------------------
 141 
 142 
 143 address LIR_Assembler::float_constant(float f) {
 144   address const_addr = __ float_constant(f);
 145   if (const_addr == NULL) {
 146     bailout("const section overflow");
 147     return __ code()->consts()->start();
 148   } else {
 149     return const_addr;
 150   }
 151 }
 152 
 153 
 154 address LIR_Assembler::double_constant(double d) {
 155   address const_addr = __ double_constant(d);
 156   if (const_addr == NULL) {
 157     bailout("const section overflow");
 158     return __ code()->consts()->start();
 159   } else {
 160     return const_addr;
 161   }
 162 }
 163 
 164 
 165 void LIR_Assembler::set_24bit_FPU() {
 166   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
 167 }
 168 
 169 void LIR_Assembler::reset_FPU() {
 170   __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
 171 }
 172 
 173 void LIR_Assembler::fpop() {
 174   __ fpop();
 175 }
 176 
 177 void LIR_Assembler::fxch(int i) {
 178   __ fxch(i);
 179 }
 180 
 181 void LIR_Assembler::fld(int i) {
 182   __ fld_s(i);
 183 }
 184 
 185 void LIR_Assembler::ffree(int i) {
 186   __ ffree(i);
 187 }
 188 
 189 void LIR_Assembler::breakpoint() {
 190   __ int3();
 191 }
 192 
 193 void LIR_Assembler::push(LIR_Opr opr) {
 194   if (opr->is_single_cpu()) {
 195     __ push_reg(opr->as_register());
 196   } else if (opr->is_double_cpu()) {
 197     NOT_LP64(__ push_reg(opr->as_register_hi()));
 198     __ push_reg(opr->as_register_lo());
 199   } else if (opr->is_stack()) {
 200     __ push_addr(frame_map()->address_for_slot(opr->single_stack_ix()));
 201   } else if (opr->is_constant()) {
 202     LIR_Const* const_opr = opr->as_constant_ptr();
 203     if (const_opr->type() == T_OBJECT || const_opr->type() == T_VALUETYPE) {
 204       __ push_oop(const_opr->as_jobject());
 205     } else if (const_opr->type() == T_INT) {
 206       __ push_jint(const_opr->as_jint());
 207     } else {
 208       ShouldNotReachHere();
 209     }
 210 
 211   } else {
 212     ShouldNotReachHere();
 213   }
 214 }
 215 
 216 void LIR_Assembler::pop(LIR_Opr opr) {
 217   if (opr->is_single_cpu()) {
 218     __ pop_reg(opr->as_register());
 219   } else {
 220     ShouldNotReachHere();
 221   }
 222 }
 223 
 224 bool LIR_Assembler::is_literal_address(LIR_Address* addr) {
 225   return addr->base()->is_illegal() && addr->index()->is_illegal();
 226 }
 227 
 228 //-------------------------------------------
 229 
 230 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 231   return as_Address(addr, rscratch1);
 232 }
 233 
 234 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 235   if (addr->base()->is_illegal()) {
 236     assert(addr->index()->is_illegal(), "must be illegal too");
 237     AddressLiteral laddr((address)addr->disp(), relocInfo::none);
 238     if (! __ reachable(laddr)) {
 239       __ movptr(tmp, laddr.addr());
 240       Address res(tmp, 0);
 241       return res;
 242     } else {
 243       return __ as_Address(laddr);
 244     }
 245   }
 246 
 247   Register base = addr->base()->as_pointer_register();
 248 
 249   if (addr->index()->is_illegal()) {
 250     return Address( base, addr->disp());
 251   } else if (addr->index()->is_cpu_register()) {
 252     Register index = addr->index()->as_pointer_register();
 253     return Address(base, index, (Address::ScaleFactor) addr->scale(), addr->disp());
 254   } else if (addr->index()->is_constant()) {
 255     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 256     assert(Assembler::is_simm32(addr_offset), "must be");
 257 
 258     return Address(base, addr_offset);
 259   } else {
 260     Unimplemented();
 261     return Address();
 262   }
 263 }
 264 
 265 
 266 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 267   Address base = as_Address(addr);
 268   return Address(base._base, base._index, base._scale, base._disp + BytesPerWord);
 269 }
 270 
 271 
 272 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 273   return as_Address(addr);
 274 }
 275 
 276 
 277 void LIR_Assembler::osr_entry() {
 278   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 279   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 280   ValueStack* entry_state = osr_entry->state();
 281   int number_of_locks = entry_state->locks_size();
 282 
 283   // we jump here if osr happens with the interpreter
 284   // state set up to continue at the beginning of the
 285   // loop that triggered osr - in particular, we have
 286   // the following registers setup:
 287   //
 288   // rcx: osr buffer
 289   //
 290 
 291   // build frame
 292   ciMethod* m = compilation()->method();
 293   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes(),
 294                  needs_stack_repair(), NULL);
 295 
 296   // OSR buffer is
 297   //
 298   // locals[nlocals-1..0]
 299   // monitors[0..number_of_locks]
 300   //
 301   // locals is a direct copy of the interpreter frame so in the osr buffer
 302   // so first slot in the local array is the last local from the interpreter
 303   // and last slot is local[0] (receiver) from the interpreter
 304   //
 305   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 306   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 307   // in the interpreter frame (the method lock if a sync method)
 308 
 309   // Initialize monitors in the compiled activation.
 310   //   rcx: pointer to osr buffer
 311   //
 312   // All other registers are dead at this point and the locals will be
 313   // copied into place by code emitted in the IR.
 314 
 315   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 316   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 317     int monitor_offset = BytesPerWord * method()->max_locals() +
 318       (BasicObjectLock::size() * BytesPerWord) * (number_of_locks - 1);
 319     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 320     // the OSR buffer using 2 word entries: first the lock and then
 321     // the oop.
 322     for (int i = 0; i < number_of_locks; i++) {
 323       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 324 #ifdef ASSERT
 325       // verify the interpreter's monitor has a non-null object
 326       {
 327         Label L;
 328         __ cmpptr(Address(OSR_buf, slot_offset + 1*BytesPerWord), (int32_t)NULL_WORD);
 329         __ jcc(Assembler::notZero, L);
 330         __ stop("locked object is NULL");
 331         __ bind(L);
 332       }
 333 #endif
 334       __ movptr(rbx, Address(OSR_buf, slot_offset + 0));
 335       __ movptr(frame_map()->address_for_monitor_lock(i), rbx);
 336       __ movptr(rbx, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 337       __ movptr(frame_map()->address_for_monitor_object(i), rbx);
 338     }
 339   }
 340 }
 341 
 342 
 343 // inline cache check; done before the frame is built.
 344 int LIR_Assembler::check_icache() {
 345   Register receiver = FrameMap::receiver_opr->as_register();
 346   Register ic_klass = IC_Klass;
 347   const int ic_cmp_size = LP64_ONLY(10) NOT_LP64(9);
 348   const bool do_post_padding = VerifyOops || UseCompressedClassPointers;
 349   if (!do_post_padding) {
 350     // insert some nops so that the verified entry point is aligned on CodeEntryAlignment
 351     __ align(CodeEntryAlignment, __ offset() + ic_cmp_size);
 352   }
 353   int offset = __ offset();
 354   __ inline_cache_check(receiver, IC_Klass);
 355   assert(__ offset() % CodeEntryAlignment == 0 || do_post_padding, "alignment must be correct");
 356   if (do_post_padding) {
 357     // force alignment after the cache check.
 358     // It's been verified to be aligned if !VerifyOops
 359     __ align(CodeEntryAlignment);
 360   }
 361   return offset;
 362 }
 363 
 364 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 365   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 366   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 367 
 368   Label L_skip_barrier;
 369   Register klass = rscratch1;
 370   Register thread = LP64_ONLY( r15_thread ) NOT_LP64( noreg );
 371   assert(thread != noreg, "x86_32 not implemented");
 372 
 373   __ mov_metadata(klass, method->holder()->constant_encoding());
 374   __ clinit_barrier(klass, thread, &L_skip_barrier /*L_fast_path*/);
 375 
 376   __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 377 
 378   __ bind(L_skip_barrier);
 379 }
 380 
 381 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo* info) {
 382   jobject o = NULL;
 383   PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
 384   __ movoop(reg, o);
 385   patching_epilog(patch, lir_patch_normal, reg, info);
 386 }
 387 
 388 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 389   Metadata* o = NULL;
 390   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 391   __ mov_metadata(reg, o);
 392   patching_epilog(patch, lir_patch_normal, reg, info);
 393 }
 394 
 395 // This specifies the rsp decrement needed to build the frame
 396 int LIR_Assembler::initial_frame_size_in_bytes() const {
 397   // if rounding, must let FrameMap know!
 398 
 399   // The frame_map records size in slots (32bit word)
 400 
 401   // subtract two words to account for return address and link
 402   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 403 }
 404 
 405 
 406 int LIR_Assembler::emit_exception_handler() {
 407   // if the last instruction is a call (typically to do a throw which
 408   // is coming at the end after block reordering) the return address
 409   // must still point into the code area in order to avoid assertion
 410   // failures when searching for the corresponding bci => add a nop
 411   // (was bug 5/14/1999 - gri)
 412   __ nop();
 413 
 414   // generate code for exception handler
 415   address handler_base = __ start_a_stub(exception_handler_size());
 416   if (handler_base == NULL) {
 417     // not enough space left for the handler
 418     bailout("exception handler overflow");
 419     return -1;
 420   }
 421 
 422   int offset = code_offset();
 423 
 424   // the exception oop and pc are in rax, and rdx
 425   // no other registers need to be preserved, so invalidate them
 426   __ invalidate_registers(false, true, true, false, true, true);
 427 
 428   // check that there is really an exception
 429   __ verify_not_null_oop(rax);
 430 
 431   // search an exception handler (rax: exception oop, rdx: throwing pc)
 432   __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 433   __ should_not_reach_here();
 434   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 435   __ end_a_stub();
 436 
 437   return offset;
 438 }
 439 
 440 
 441 // Emit the code to remove the frame from the stack in the exception
 442 // unwind path.
 443 int LIR_Assembler::emit_unwind_handler() {
 444 #ifndef PRODUCT
 445   if (CommentedAssembly) {
 446     _masm->block_comment("Unwind handler");
 447   }
 448 #endif
 449 
 450   int offset = code_offset();
 451 
 452   // Fetch the exception from TLS and clear out exception related thread state
 453   Register thread = NOT_LP64(rsi) LP64_ONLY(r15_thread);
 454   NOT_LP64(__ get_thread(rsi));
 455   __ movptr(rax, Address(thread, JavaThread::exception_oop_offset()));
 456   __ movptr(Address(thread, JavaThread::exception_oop_offset()), (intptr_t)NULL_WORD);
 457   __ movptr(Address(thread, JavaThread::exception_pc_offset()), (intptr_t)NULL_WORD);
 458 
 459   __ bind(_unwind_handler_entry);
 460   __ verify_not_null_oop(rax);
 461   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 462     __ mov(rbx, rax);  // Preserve the exception (rbx is always callee-saved)
 463   }
 464 
 465   // Preform needed unlocking
 466   MonitorExitStub* stub = NULL;
 467   if (method()->is_synchronized()) {
 468     monitor_address(0, FrameMap::rax_opr);
 469     stub = new MonitorExitStub(FrameMap::rax_opr, true, 0);
 470     __ unlock_object(rdi, rsi, rax, *stub->entry());
 471     __ bind(*stub->continuation());
 472   }
 473 
 474   if (compilation()->env()->dtrace_method_probes()) {
 475 #ifdef _LP64
 476     __ mov(rdi, r15_thread);
 477     __ mov_metadata(rsi, method()->constant_encoding());
 478 #else
 479     __ get_thread(rax);
 480     __ movptr(Address(rsp, 0), rax);
 481     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
 482 #endif
 483     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 484   }
 485 
 486   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 487     __ mov(rax, rbx);  // Restore the exception
 488   }
 489 
 490   // remove the activation and dispatch to the unwind handler
 491   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 492   __ jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 493 
 494   // Emit the slow path assembly
 495   if (stub != NULL) {
 496     stub->emit_code(this);
 497   }
 498 
 499   return offset;
 500 }
 501 
 502 
 503 int LIR_Assembler::emit_deopt_handler() {
 504   // if the last instruction is a call (typically to do a throw which
 505   // is coming at the end after block reordering) the return address
 506   // must still point into the code area in order to avoid assertion
 507   // failures when searching for the corresponding bci => add a nop
 508   // (was bug 5/14/1999 - gri)
 509   __ nop();
 510 
 511   // generate code for exception handler
 512   address handler_base = __ start_a_stub(deopt_handler_size());
 513   if (handler_base == NULL) {
 514     // not enough space left for the handler
 515     bailout("deopt handler overflow");
 516     return -1;
 517   }
 518 
 519   int offset = code_offset();
 520   InternalAddress here(__ pc());
 521 
 522   __ pushptr(here.addr());
 523   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 524   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 525   __ end_a_stub();
 526 
 527   return offset;
 528 }
 529 
 530 
 531 void LIR_Assembler::return_op(LIR_Opr result) {
 532   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 533   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 534     assert(result->fpu() == 0, "result must already be on TOS");
 535   }
 536 
 537   ciMethod* method = compilation()->method();
 538   if (ValueTypeReturnedAsFields && method->signature()->returns_never_null()) {
 539     ciType* return_type = method->return_type();
 540     if (return_type->is_valuetype()) {
 541       ciValueKlass* vk = return_type->as_value_klass();
 542       if (vk->can_be_returned_as_fields()) {
 543 #ifndef _LP64
 544         Unimplemented();
 545 #else
 546         address unpack_handler = vk->unpack_handler();
 547         assert(unpack_handler != NULL, "must be");
 548         __ call(RuntimeAddress(unpack_handler));
 549         // At this point, rax points to the value object (for interpreter or C1 caller).
 550         // The fields of the object are copied into registers (for C2 caller).
 551 #endif
 552       }
 553     }
 554   }
 555 
 556   // Pop the stack before the safepoint code
 557   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 558 
 559   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 560     __ reserved_stack_check();
 561   }
 562 
 563   bool result_is_oop = result->is_valid() ? result->is_oop() : false;
 564 
 565   // Note: we do not need to round double result; float result has the right precision
 566   // the poll sets the condition code, but no data registers
 567 
 568   if (SafepointMechanism::uses_thread_local_poll()) {
 569 #ifdef _LP64
 570     const Register poll_addr = rscratch1;
 571     __ movptr(poll_addr, Address(r15_thread, Thread::polling_page_offset()));
 572 #else
 573     const Register poll_addr = rbx;
 574     assert(FrameMap::is_caller_save_register(poll_addr), "will overwrite");
 575     __ get_thread(poll_addr);
 576     __ movptr(poll_addr, Address(poll_addr, Thread::polling_page_offset()));
 577 #endif
 578     __ relocate(relocInfo::poll_return_type);
 579     __ testl(rax, Address(poll_addr, 0));
 580   } else {
 581     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_return_type);
 582 
 583     if (Assembler::is_polling_page_far()) {
 584       __ lea(rscratch1, polling_page);
 585       __ relocate(relocInfo::poll_return_type);
 586       __ testl(rax, Address(rscratch1, 0));
 587     } else {
 588       __ testl(rax, polling_page);
 589     }
 590   }
 591   __ ret(0);
 592 }
 593 
 594 
 595 int LIR_Assembler::store_value_type_fields_to_buf(ciValueKlass* vk) {
 596   return (__ store_value_type_fields_to_buf(vk, false));
 597 }
 598 
 599 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 600   guarantee(info != NULL, "Shouldn't be NULL");
 601   int offset = __ offset();
 602   if (SafepointMechanism::uses_thread_local_poll()) {
 603 #ifdef _LP64
 604     const Register poll_addr = rscratch1;
 605     __ movptr(poll_addr, Address(r15_thread, Thread::polling_page_offset()));
 606 #else
 607     assert(tmp->is_cpu_register(), "needed");
 608     const Register poll_addr = tmp->as_register();
 609     __ get_thread(poll_addr);
 610     __ movptr(poll_addr, Address(poll_addr, in_bytes(Thread::polling_page_offset())));
 611 #endif
 612     add_debug_info_for_branch(info);
 613     __ relocate(relocInfo::poll_type);
 614     address pre_pc = __ pc();
 615     __ testl(rax, Address(poll_addr, 0));
 616     address post_pc = __ pc();
 617     guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
 618   } else {
 619     AddressLiteral polling_page(os::get_polling_page(), relocInfo::poll_type);
 620     if (Assembler::is_polling_page_far()) {
 621       __ lea(rscratch1, polling_page);
 622       offset = __ offset();
 623       add_debug_info_for_branch(info);
 624       __ relocate(relocInfo::poll_type);
 625       __ testl(rax, Address(rscratch1, 0));
 626     } else {
 627       add_debug_info_for_branch(info);
 628       __ testl(rax, polling_page);
 629     }
 630   }
 631   return offset;
 632 }
 633 
 634 
 635 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 636   if (from_reg != to_reg) __ mov(to_reg, from_reg);
 637 }
 638 
 639 void LIR_Assembler::swap_reg(Register a, Register b) {
 640   __ xchgptr(a, b);
 641 }
 642 
 643 
 644 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 645   assert(src->is_constant(), "should not call otherwise");
 646   assert(dest->is_register(), "should not call otherwise");
 647   LIR_Const* c = src->as_constant_ptr();
 648 
 649   switch (c->type()) {
 650     case T_INT: {
 651       assert(patch_code == lir_patch_none, "no patching handled here");
 652       __ movl(dest->as_register(), c->as_jint());
 653       break;
 654     }
 655 
 656     case T_ADDRESS: {
 657       assert(patch_code == lir_patch_none, "no patching handled here");
 658       __ movptr(dest->as_register(), c->as_jint());
 659       break;
 660     }
 661 
 662     case T_LONG: {
 663       assert(patch_code == lir_patch_none, "no patching handled here");
 664 #ifdef _LP64
 665       __ movptr(dest->as_register_lo(), (intptr_t)c->as_jlong());
 666 #else
 667       __ movptr(dest->as_register_lo(), c->as_jint_lo());
 668       __ movptr(dest->as_register_hi(), c->as_jint_hi());
 669 #endif // _LP64
 670       break;
 671     }
 672 
 673     case T_VALUETYPE: // Fall through
 674     case T_OBJECT: {
 675       if (patch_code != lir_patch_none) {
 676         jobject2reg_with_patching(dest->as_register(), info);
 677       } else {
 678         __ movoop(dest->as_register(), c->as_jobject());
 679       }
 680       break;
 681     }
 682 
 683     case T_METADATA: {
 684       if (patch_code != lir_patch_none) {
 685         klass2reg_with_patching(dest->as_register(), info);
 686       } else {
 687         __ mov_metadata(dest->as_register(), c->as_metadata());
 688       }
 689       break;
 690     }
 691 
 692     case T_FLOAT: {
 693       if (dest->is_single_xmm()) {
 694         if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_float()) {
 695           __ xorps(dest->as_xmm_float_reg(), dest->as_xmm_float_reg());
 696         } else {
 697           __ movflt(dest->as_xmm_float_reg(),
 698                    InternalAddress(float_constant(c->as_jfloat())));
 699         }
 700       } else {
 701         assert(dest->is_single_fpu(), "must be");
 702         assert(dest->fpu_regnr() == 0, "dest must be TOS");
 703         if (c->is_zero_float()) {
 704           __ fldz();
 705         } else if (c->is_one_float()) {
 706           __ fld1();
 707         } else {
 708           __ fld_s (InternalAddress(float_constant(c->as_jfloat())));
 709         }
 710       }
 711       break;
 712     }
 713 
 714     case T_DOUBLE: {
 715       if (dest->is_double_xmm()) {
 716         if (LP64_ONLY(UseAVX <= 2 &&) c->is_zero_double()) {
 717           __ xorpd(dest->as_xmm_double_reg(), dest->as_xmm_double_reg());
 718         } else {
 719           __ movdbl(dest->as_xmm_double_reg(),
 720                     InternalAddress(double_constant(c->as_jdouble())));
 721         }
 722       } else {
 723         assert(dest->is_double_fpu(), "must be");
 724         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
 725         if (c->is_zero_double()) {
 726           __ fldz();
 727         } else if (c->is_one_double()) {
 728           __ fld1();
 729         } else {
 730           __ fld_d (InternalAddress(double_constant(c->as_jdouble())));
 731         }
 732       }
 733       break;
 734     }
 735 
 736     default:
 737       ShouldNotReachHere();
 738   }
 739 }
 740 
 741 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 742   assert(src->is_constant(), "should not call otherwise");
 743   assert(dest->is_stack(), "should not call otherwise");
 744   LIR_Const* c = src->as_constant_ptr();
 745 
 746   switch (c->type()) {
 747     case T_INT:  // fall through
 748     case T_FLOAT:
 749       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 750       break;
 751 
 752     case T_ADDRESS:
 753       __ movptr(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jint_bits());
 754       break;
 755 
 756     case T_VALUETYPE: // Fall through
 757     case T_OBJECT:
 758       __ movoop(frame_map()->address_for_slot(dest->single_stack_ix()), c->as_jobject());
 759       break;
 760 
 761     case T_LONG:  // fall through
 762     case T_DOUBLE:
 763 #ifdef _LP64
 764       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 765                                             lo_word_offset_in_bytes), (intptr_t)c->as_jlong_bits());
 766 #else
 767       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 768                                               lo_word_offset_in_bytes), c->as_jint_lo_bits());
 769       __ movptr(frame_map()->address_for_slot(dest->double_stack_ix(),
 770                                               hi_word_offset_in_bytes), c->as_jint_hi_bits());
 771 #endif // _LP64
 772       break;
 773 
 774     default:
 775       ShouldNotReachHere();
 776   }
 777 }
 778 
 779 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 780   assert(src->is_constant(), "should not call otherwise");
 781   assert(dest->is_address(), "should not call otherwise");
 782   LIR_Const* c = src->as_constant_ptr();
 783   LIR_Address* addr = dest->as_address_ptr();
 784 
 785   int null_check_here = code_offset();
 786   switch (type) {
 787     case T_INT:    // fall through
 788     case T_FLOAT:
 789       __ movl(as_Address(addr), c->as_jint_bits());
 790       break;
 791 
 792     case T_ADDRESS:
 793       __ movptr(as_Address(addr), c->as_jint_bits());
 794       break;
 795 
 796     case T_VALUETYPE: // fall through
 797     case T_OBJECT:  // fall through
 798     case T_ARRAY:
 799       if (c->as_jobject() == NULL) {
 800         if (UseCompressedOops && !wide) {
 801           __ movl(as_Address(addr), (int32_t)NULL_WORD);
 802         } else {
 803 #ifdef _LP64
 804           __ xorptr(rscratch1, rscratch1);
 805           null_check_here = code_offset();
 806           __ movptr(as_Address(addr), rscratch1);
 807 #else
 808           __ movptr(as_Address(addr), NULL_WORD);
 809 #endif
 810         }
 811       } else {
 812         if (is_literal_address(addr)) {
 813           ShouldNotReachHere();
 814           __ movoop(as_Address(addr, noreg), c->as_jobject());
 815         } else {
 816 #ifdef _LP64
 817           __ movoop(rscratch1, c->as_jobject());
 818           if (UseCompressedOops && !wide) {
 819             __ encode_heap_oop(rscratch1);
 820             null_check_here = code_offset();
 821             __ movl(as_Address_lo(addr), rscratch1);
 822           } else {
 823             null_check_here = code_offset();
 824             __ movptr(as_Address_lo(addr), rscratch1);
 825           }
 826 #else
 827           __ movoop(as_Address(addr), c->as_jobject());
 828 #endif
 829         }
 830       }
 831       break;
 832 
 833     case T_LONG:    // fall through
 834     case T_DOUBLE:
 835 #ifdef _LP64
 836       if (is_literal_address(addr)) {
 837         ShouldNotReachHere();
 838         __ movptr(as_Address(addr, r15_thread), (intptr_t)c->as_jlong_bits());
 839       } else {
 840         __ movptr(r10, (intptr_t)c->as_jlong_bits());
 841         null_check_here = code_offset();
 842         __ movptr(as_Address_lo(addr), r10);
 843       }
 844 #else
 845       // Always reachable in 32bit so this doesn't produce useless move literal
 846       __ movptr(as_Address_hi(addr), c->as_jint_hi_bits());
 847       __ movptr(as_Address_lo(addr), c->as_jint_lo_bits());
 848 #endif // _LP64
 849       break;
 850 
 851     case T_BOOLEAN: // fall through
 852     case T_BYTE:
 853       __ movb(as_Address(addr), c->as_jint() & 0xFF);
 854       break;
 855 
 856     case T_CHAR:    // fall through
 857     case T_SHORT:
 858       __ movw(as_Address(addr), c->as_jint() & 0xFFFF);
 859       break;
 860 
 861     default:
 862       ShouldNotReachHere();
 863   };
 864 
 865   if (info != NULL) {
 866     add_debug_info_for_null_check(null_check_here, info);
 867   }
 868 }
 869 
 870 
 871 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 872   assert(src->is_register(), "should not call otherwise");
 873   assert(dest->is_register(), "should not call otherwise");
 874 
 875   // move between cpu-registers
 876   if (dest->is_single_cpu()) {
 877 #ifdef _LP64
 878     if (src->type() == T_LONG) {
 879       // Can do LONG -> OBJECT
 880       move_regs(src->as_register_lo(), dest->as_register());
 881       return;
 882     }
 883 #endif
 884     assert(src->is_single_cpu(), "must match");
 885     if (src->type() == T_OBJECT || src->type() == T_VALUETYPE) {
 886       __ verify_oop(src->as_register());
 887     }
 888     move_regs(src->as_register(), dest->as_register());
 889 
 890   } else if (dest->is_double_cpu()) {
 891 #ifdef _LP64
 892     if (src->type() == T_OBJECT || src->type() == T_ARRAY || src->type() == T_VALUETYPE) {
 893       // Surprising to me but we can see move of a long to t_object
 894       __ verify_oop(src->as_register());
 895       move_regs(src->as_register(), dest->as_register_lo());
 896       return;
 897     }
 898 #endif
 899     assert(src->is_double_cpu(), "must match");
 900     Register f_lo = src->as_register_lo();
 901     Register f_hi = src->as_register_hi();
 902     Register t_lo = dest->as_register_lo();
 903     Register t_hi = dest->as_register_hi();
 904 #ifdef _LP64
 905     assert(f_hi == f_lo, "must be same");
 906     assert(t_hi == t_lo, "must be same");
 907     move_regs(f_lo, t_lo);
 908 #else
 909     assert(f_lo != f_hi && t_lo != t_hi, "invalid register allocation");
 910 
 911 
 912     if (f_lo == t_hi && f_hi == t_lo) {
 913       swap_reg(f_lo, f_hi);
 914     } else if (f_hi == t_lo) {
 915       assert(f_lo != t_hi, "overwriting register");
 916       move_regs(f_hi, t_hi);
 917       move_regs(f_lo, t_lo);
 918     } else {
 919       assert(f_hi != t_lo, "overwriting register");
 920       move_regs(f_lo, t_lo);
 921       move_regs(f_hi, t_hi);
 922     }
 923 #endif // LP64
 924 
 925     // special moves from fpu-register to xmm-register
 926     // necessary for method results
 927   } else if (src->is_single_xmm() && !dest->is_single_xmm()) {
 928     __ movflt(Address(rsp, 0), src->as_xmm_float_reg());
 929     __ fld_s(Address(rsp, 0));
 930   } else if (src->is_double_xmm() && !dest->is_double_xmm()) {
 931     __ movdbl(Address(rsp, 0), src->as_xmm_double_reg());
 932     __ fld_d(Address(rsp, 0));
 933   } else if (dest->is_single_xmm() && !src->is_single_xmm()) {
 934     __ fstp_s(Address(rsp, 0));
 935     __ movflt(dest->as_xmm_float_reg(), Address(rsp, 0));
 936   } else if (dest->is_double_xmm() && !src->is_double_xmm()) {
 937     __ fstp_d(Address(rsp, 0));
 938     __ movdbl(dest->as_xmm_double_reg(), Address(rsp, 0));
 939 
 940     // move between xmm-registers
 941   } else if (dest->is_single_xmm()) {
 942     assert(src->is_single_xmm(), "must match");
 943     __ movflt(dest->as_xmm_float_reg(), src->as_xmm_float_reg());
 944   } else if (dest->is_double_xmm()) {
 945     assert(src->is_double_xmm(), "must match");
 946     __ movdbl(dest->as_xmm_double_reg(), src->as_xmm_double_reg());
 947 
 948     // move between fpu-registers (no instruction necessary because of fpu-stack)
 949   } else if (dest->is_single_fpu() || dest->is_double_fpu()) {
 950     assert(src->is_single_fpu() || src->is_double_fpu(), "must match");
 951     assert(src->fpu() == dest->fpu(), "currently should be nothing to do");
 952   } else {
 953     ShouldNotReachHere();
 954   }
 955 }
 956 
 957 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 958   assert(src->is_register(), "should not call otherwise");
 959   assert(dest->is_stack(), "should not call otherwise");
 960 
 961   if (src->is_single_cpu()) {
 962     Address dst = frame_map()->address_for_slot(dest->single_stack_ix());
 963     if (type == T_OBJECT || type == T_ARRAY || type == T_VALUETYPE) {
 964       __ verify_oop(src->as_register());
 965       __ movptr (dst, src->as_register());
 966     } else if (type == T_METADATA) {
 967       __ movptr (dst, src->as_register());
 968     } else {
 969       __ movl (dst, src->as_register());
 970     }
 971 
 972   } else if (src->is_double_cpu()) {
 973     Address dstLO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 974     Address dstHI = frame_map()->address_for_slot(dest->double_stack_ix(), hi_word_offset_in_bytes);
 975     __ movptr (dstLO, src->as_register_lo());
 976     NOT_LP64(__ movptr (dstHI, src->as_register_hi()));
 977 
 978   } else if (src->is_single_xmm()) {
 979     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 980     __ movflt(dst_addr, src->as_xmm_float_reg());
 981 
 982   } else if (src->is_double_xmm()) {
 983     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 984     __ movdbl(dst_addr, src->as_xmm_double_reg());
 985 
 986   } else if (src->is_single_fpu()) {
 987     assert(src->fpu_regnr() == 0, "argument must be on TOS");
 988     Address dst_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 989     if (pop_fpu_stack)     __ fstp_s (dst_addr);
 990     else                   __ fst_s  (dst_addr);
 991 
 992   } else if (src->is_double_fpu()) {
 993     assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
 994     Address dst_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 995     if (pop_fpu_stack)     __ fstp_d (dst_addr);
 996     else                   __ fst_d  (dst_addr);
 997 
 998   } else {
 999     ShouldNotReachHere();
1000   }
1001 }
1002 
1003 
1004 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
1005   LIR_Address* to_addr = dest->as_address_ptr();
1006   PatchingStub* patch = NULL;
1007   Register compressed_src = rscratch1;
1008 
1009   if (type == T_ARRAY || type == T_OBJECT || type == T_VALUETYPE) {
1010     __ verify_oop(src->as_register());
1011 #ifdef _LP64
1012     if (UseCompressedOops && !wide) {
1013       __ movptr(compressed_src, src->as_register());
1014       __ encode_heap_oop(compressed_src);
1015       if (patch_code != lir_patch_none) {
1016         info->oop_map()->set_narrowoop(compressed_src->as_VMReg());
1017       }
1018     }
1019 #endif
1020   }
1021 
1022   if (patch_code != lir_patch_none) {
1023     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1024     Address toa = as_Address(to_addr);
1025     assert(toa.disp() != 0, "must have");
1026   }
1027 
1028   int null_check_here = code_offset();
1029   switch (type) {
1030     case T_FLOAT: {
1031       if (src->is_single_xmm()) {
1032         __ movflt(as_Address(to_addr), src->as_xmm_float_reg());
1033       } else {
1034         assert(src->is_single_fpu(), "must be");
1035         assert(src->fpu_regnr() == 0, "argument must be on TOS");
1036         if (pop_fpu_stack)      __ fstp_s(as_Address(to_addr));
1037         else                    __ fst_s (as_Address(to_addr));
1038       }
1039       break;
1040     }
1041 
1042     case T_DOUBLE: {
1043       if (src->is_double_xmm()) {
1044         __ movdbl(as_Address(to_addr), src->as_xmm_double_reg());
1045       } else {
1046         assert(src->is_double_fpu(), "must be");
1047         assert(src->fpu_regnrLo() == 0, "argument must be on TOS");
1048         if (pop_fpu_stack)      __ fstp_d(as_Address(to_addr));
1049         else                    __ fst_d (as_Address(to_addr));
1050       }
1051       break;
1052     }
1053 
1054     case T_VALUETYPE: // fall through
1055     case T_ARRAY:   // fall through
1056     case T_OBJECT:  // fall through
1057       if (UseCompressedOops && !wide) {
1058         __ movl(as_Address(to_addr), compressed_src);
1059       } else {
1060         __ movptr(as_Address(to_addr), src->as_register());
1061       }
1062       break;
1063     case T_METADATA:
1064       // We get here to store a method pointer to the stack to pass to
1065       // a dtrace runtime call. This can't work on 64 bit with
1066       // compressed klass ptrs: T_METADATA can be a compressed klass
1067       // ptr or a 64 bit method pointer.
1068       LP64_ONLY(ShouldNotReachHere());
1069       __ movptr(as_Address(to_addr), src->as_register());
1070       break;
1071     case T_ADDRESS:
1072       __ movptr(as_Address(to_addr), src->as_register());
1073       break;
1074     case T_INT:
1075       __ movl(as_Address(to_addr), src->as_register());
1076       break;
1077 
1078     case T_LONG: {
1079       Register from_lo = src->as_register_lo();
1080       Register from_hi = src->as_register_hi();
1081 #ifdef _LP64
1082       __ movptr(as_Address_lo(to_addr), from_lo);
1083 #else
1084       Register base = to_addr->base()->as_register();
1085       Register index = noreg;
1086       if (to_addr->index()->is_register()) {
1087         index = to_addr->index()->as_register();
1088       }
1089       if (base == from_lo || index == from_lo) {
1090         assert(base != from_hi, "can't be");
1091         assert(index == noreg || (index != base && index != from_hi), "can't handle this");
1092         __ movl(as_Address_hi(to_addr), from_hi);
1093         if (patch != NULL) {
1094           patching_epilog(patch, lir_patch_high, base, info);
1095           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1096           patch_code = lir_patch_low;
1097         }
1098         __ movl(as_Address_lo(to_addr), from_lo);
1099       } else {
1100         assert(index == noreg || (index != base && index != from_lo), "can't handle this");
1101         __ movl(as_Address_lo(to_addr), from_lo);
1102         if (patch != NULL) {
1103           patching_epilog(patch, lir_patch_low, base, info);
1104           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1105           patch_code = lir_patch_high;
1106         }
1107         __ movl(as_Address_hi(to_addr), from_hi);
1108       }
1109 #endif // _LP64
1110       break;
1111     }
1112 
1113     case T_BYTE:    // fall through
1114     case T_BOOLEAN: {
1115       Register src_reg = src->as_register();
1116       Address dst_addr = as_Address(to_addr);
1117       assert(VM_Version::is_P6() || src_reg->has_byte_register(), "must use byte registers if not P6");
1118       __ movb(dst_addr, src_reg);
1119       break;
1120     }
1121 
1122     case T_CHAR:    // fall through
1123     case T_SHORT:
1124       __ movw(as_Address(to_addr), src->as_register());
1125       break;
1126 
1127     default:
1128       ShouldNotReachHere();
1129   }
1130   if (info != NULL) {
1131     add_debug_info_for_null_check(null_check_here, info);
1132   }
1133 
1134   if (patch_code != lir_patch_none) {
1135     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
1136   }
1137 }
1138 
1139 
1140 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1141   assert(src->is_stack(), "should not call otherwise");
1142   assert(dest->is_register(), "should not call otherwise");
1143 
1144   if (dest->is_single_cpu()) {
1145     if (type == T_ARRAY || type == T_OBJECT || type == T_VALUETYPE) {
1146       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1147       __ verify_oop(dest->as_register());
1148     } else if (type == T_METADATA) {
1149       __ movptr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1150     } else {
1151       __ movl(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
1152     }
1153 
1154   } else if (dest->is_double_cpu()) {
1155     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
1156     Address src_addr_HI = frame_map()->address_for_slot(src->double_stack_ix(), hi_word_offset_in_bytes);
1157     __ movptr(dest->as_register_lo(), src_addr_LO);
1158     NOT_LP64(__ movptr(dest->as_register_hi(), src_addr_HI));
1159 
1160   } else if (dest->is_single_xmm()) {
1161     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1162     __ movflt(dest->as_xmm_float_reg(), src_addr);
1163 
1164   } else if (dest->is_double_xmm()) {
1165     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1166     __ movdbl(dest->as_xmm_double_reg(), src_addr);
1167 
1168   } else if (dest->is_single_fpu()) {
1169     assert(dest->fpu_regnr() == 0, "dest must be TOS");
1170     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
1171     __ fld_s(src_addr);
1172 
1173   } else if (dest->is_double_fpu()) {
1174     assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1175     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
1176     __ fld_d(src_addr);
1177 
1178   } else {
1179     ShouldNotReachHere();
1180   }
1181 }
1182 
1183 
1184 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1185   if (src->is_single_stack()) {
1186     if (type == T_OBJECT || type == T_ARRAY || type == T_VALUETYPE) {
1187       __ pushptr(frame_map()->address_for_slot(src ->single_stack_ix()));
1188       __ popptr (frame_map()->address_for_slot(dest->single_stack_ix()));
1189     } else {
1190 #ifndef _LP64
1191       __ pushl(frame_map()->address_for_slot(src ->single_stack_ix()));
1192       __ popl (frame_map()->address_for_slot(dest->single_stack_ix()));
1193 #else
1194       //no pushl on 64bits
1195       __ movl(rscratch1, frame_map()->address_for_slot(src ->single_stack_ix()));
1196       __ movl(frame_map()->address_for_slot(dest->single_stack_ix()), rscratch1);
1197 #endif
1198     }
1199 
1200   } else if (src->is_double_stack()) {
1201 #ifdef _LP64
1202     __ pushptr(frame_map()->address_for_slot(src ->double_stack_ix()));
1203     __ popptr (frame_map()->address_for_slot(dest->double_stack_ix()));
1204 #else
1205     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 0));
1206     // push and pop the part at src + wordSize, adding wordSize for the previous push
1207     __ pushl(frame_map()->address_for_slot(src ->double_stack_ix(), 2 * wordSize));
1208     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 2 * wordSize));
1209     __ popl (frame_map()->address_for_slot(dest->double_stack_ix(), 0));
1210 #endif // _LP64
1211 
1212   } else {
1213     ShouldNotReachHere();
1214   }
1215 }
1216 
1217 
1218 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
1219   assert(src->is_address(), "should not call otherwise");
1220   assert(dest->is_register(), "should not call otherwise");
1221 
1222   LIR_Address* addr = src->as_address_ptr();
1223   Address from_addr = as_Address(addr);
1224 
1225   if (addr->base()->type() == T_OBJECT || addr->base()->type() == T_VALUETYPE) {
1226     __ verify_oop(addr->base()->as_pointer_register());
1227   }
1228 
1229   switch (type) {
1230     case T_BOOLEAN: // fall through
1231     case T_BYTE:    // fall through
1232     case T_CHAR:    // fall through
1233     case T_SHORT:
1234       if (!VM_Version::is_P6() && !from_addr.uses(dest->as_register())) {
1235         // on pre P6 processors we may get partial register stalls
1236         // so blow away the value of to_rinfo before loading a
1237         // partial word into it.  Do it here so that it precedes
1238         // the potential patch point below.
1239         __ xorptr(dest->as_register(), dest->as_register());
1240       }
1241       break;
1242    default:
1243      break;
1244   }
1245 
1246   PatchingStub* patch = NULL;
1247   if (patch_code != lir_patch_none) {
1248     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1249     assert(from_addr.disp() != 0, "must have");
1250   }
1251   if (info != NULL) {
1252     add_debug_info_for_null_check_here(info);
1253   }
1254 
1255   switch (type) {
1256     case T_FLOAT: {
1257       if (dest->is_single_xmm()) {
1258         __ movflt(dest->as_xmm_float_reg(), from_addr);
1259       } else {
1260         assert(dest->is_single_fpu(), "must be");
1261         assert(dest->fpu_regnr() == 0, "dest must be TOS");
1262         __ fld_s(from_addr);
1263       }
1264       break;
1265     }
1266 
1267     case T_DOUBLE: {
1268       if (dest->is_double_xmm()) {
1269         __ movdbl(dest->as_xmm_double_reg(), from_addr);
1270       } else {
1271         assert(dest->is_double_fpu(), "must be");
1272         assert(dest->fpu_regnrLo() == 0, "dest must be TOS");
1273         __ fld_d(from_addr);
1274       }
1275       break;
1276     }
1277 
1278     case T_VALUETYPE: // fall through
1279     case T_OBJECT:  // fall through
1280     case T_ARRAY:   // fall through
1281       if (UseCompressedOops && !wide) {
1282         __ movl(dest->as_register(), from_addr);
1283       } else {
1284         __ movptr(dest->as_register(), from_addr);
1285       }
1286       break;
1287 
1288     case T_ADDRESS:
1289       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1290         __ movl(dest->as_register(), from_addr);
1291       } else {
1292         __ movptr(dest->as_register(), from_addr);
1293       }
1294       break;
1295     case T_INT:
1296       __ movl(dest->as_register(), from_addr);
1297       break;
1298 
1299     case T_LONG: {
1300       Register to_lo = dest->as_register_lo();
1301       Register to_hi = dest->as_register_hi();
1302 #ifdef _LP64
1303       __ movptr(to_lo, as_Address_lo(addr));
1304 #else
1305       Register base = addr->base()->as_register();
1306       Register index = noreg;
1307       if (addr->index()->is_register()) {
1308         index = addr->index()->as_register();
1309       }
1310       if ((base == to_lo && index == to_hi) ||
1311           (base == to_hi && index == to_lo)) {
1312         // addresses with 2 registers are only formed as a result of
1313         // array access so this code will never have to deal with
1314         // patches or null checks.
1315         assert(info == NULL && patch == NULL, "must be");
1316         __ lea(to_hi, as_Address(addr));
1317         __ movl(to_lo, Address(to_hi, 0));
1318         __ movl(to_hi, Address(to_hi, BytesPerWord));
1319       } else if (base == to_lo || index == to_lo) {
1320         assert(base != to_hi, "can't be");
1321         assert(index == noreg || (index != base && index != to_hi), "can't handle this");
1322         __ movl(to_hi, as_Address_hi(addr));
1323         if (patch != NULL) {
1324           patching_epilog(patch, lir_patch_high, base, info);
1325           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1326           patch_code = lir_patch_low;
1327         }
1328         __ movl(to_lo, as_Address_lo(addr));
1329       } else {
1330         assert(index == noreg || (index != base && index != to_lo), "can't handle this");
1331         __ movl(to_lo, as_Address_lo(addr));
1332         if (patch != NULL) {
1333           patching_epilog(patch, lir_patch_low, base, info);
1334           patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1335           patch_code = lir_patch_high;
1336         }
1337         __ movl(to_hi, as_Address_hi(addr));
1338       }
1339 #endif // _LP64
1340       break;
1341     }
1342 
1343     case T_BOOLEAN: // fall through
1344     case T_BYTE: {
1345       Register dest_reg = dest->as_register();
1346       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1347       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1348         __ movsbl(dest_reg, from_addr);
1349       } else {
1350         __ movb(dest_reg, from_addr);
1351         __ shll(dest_reg, 24);
1352         __ sarl(dest_reg, 24);
1353       }
1354       break;
1355     }
1356 
1357     case T_CHAR: {
1358       Register dest_reg = dest->as_register();
1359       assert(VM_Version::is_P6() || dest_reg->has_byte_register(), "must use byte registers if not P6");
1360       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1361         __ movzwl(dest_reg, from_addr);
1362       } else {
1363         __ movw(dest_reg, from_addr);
1364       }
1365       break;
1366     }
1367 
1368     case T_SHORT: {
1369       Register dest_reg = dest->as_register();
1370       if (VM_Version::is_P6() || from_addr.uses(dest_reg)) {
1371         __ movswl(dest_reg, from_addr);
1372       } else {
1373         __ movw(dest_reg, from_addr);
1374         __ shll(dest_reg, 16);
1375         __ sarl(dest_reg, 16);
1376       }
1377       break;
1378     }
1379 
1380     default:
1381       ShouldNotReachHere();
1382   }
1383 
1384   if (patch != NULL) {
1385     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
1386   }
1387 
1388   if (type == T_ARRAY || type == T_OBJECT || type == T_VALUETYPE) {
1389 #ifdef _LP64
1390     if (UseCompressedOops && !wide) {
1391       __ decode_heap_oop(dest->as_register());
1392     }
1393 #endif
1394 
1395     // Load barrier has not yet been applied, so ZGC can't verify the oop here
1396     if (!UseZGC) {
1397       __ verify_oop(dest->as_register());
1398     }
1399   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1400 #ifdef _LP64
1401     if (UseCompressedClassPointers) {
1402       __ andl(dest->as_register(), oopDesc::compressed_klass_mask());
1403       __ decode_klass_not_null(dest->as_register());
1404     } else {
1405       __ shlq(dest->as_register(), oopDesc::storage_props_nof_bits);
1406       __ shrq(dest->as_register(), oopDesc::storage_props_nof_bits);
1407     }
1408 #else
1409     __ andl(dest->as_register(), oopDesc::wide_klass_mask());
1410 #endif
1411   }
1412 }
1413 
1414 
1415 NEEDS_CLEANUP; // This could be static?
1416 Address::ScaleFactor LIR_Assembler::array_element_size(BasicType type) const {
1417   int elem_size = type2aelembytes(type);
1418   switch (elem_size) {
1419     case 1: return Address::times_1;
1420     case 2: return Address::times_2;
1421     case 4: return Address::times_4;
1422     case 8: return Address::times_8;
1423   }
1424   ShouldNotReachHere();
1425   return Address::no_scale;
1426 }
1427 
1428 
1429 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1430   switch (op->code()) {
1431     case lir_idiv:
1432     case lir_irem:
1433       arithmetic_idiv(op->code(),
1434                       op->in_opr1(),
1435                       op->in_opr2(),
1436                       op->in_opr3(),
1437                       op->result_opr(),
1438                       op->info());
1439       break;
1440     case lir_fmad:
1441       __ fmad(op->result_opr()->as_xmm_double_reg(),
1442               op->in_opr1()->as_xmm_double_reg(),
1443               op->in_opr2()->as_xmm_double_reg(),
1444               op->in_opr3()->as_xmm_double_reg());
1445       break;
1446     case lir_fmaf:
1447       __ fmaf(op->result_opr()->as_xmm_float_reg(),
1448               op->in_opr1()->as_xmm_float_reg(),
1449               op->in_opr2()->as_xmm_float_reg(),
1450               op->in_opr3()->as_xmm_float_reg());
1451       break;
1452     default:      ShouldNotReachHere(); break;
1453   }
1454 }
1455 
1456 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1457 #ifdef ASSERT
1458   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1459   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1460   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1461 #endif
1462 
1463   if (op->cond() == lir_cond_always) {
1464     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1465     __ jmp (*(op->label()));
1466   } else {
1467     Assembler::Condition acond = Assembler::zero;
1468     if (op->code() == lir_cond_float_branch) {
1469       assert(op->ublock() != NULL, "must have unordered successor");
1470       __ jcc(Assembler::parity, *(op->ublock()->label()));
1471       switch(op->cond()) {
1472         case lir_cond_equal:        acond = Assembler::equal;      break;
1473         case lir_cond_notEqual:     acond = Assembler::notEqual;   break;
1474         case lir_cond_less:         acond = Assembler::below;      break;
1475         case lir_cond_lessEqual:    acond = Assembler::belowEqual; break;
1476         case lir_cond_greaterEqual: acond = Assembler::aboveEqual; break;
1477         case lir_cond_greater:      acond = Assembler::above;      break;
1478         default:                         ShouldNotReachHere();
1479       }
1480     } else {
1481       switch (op->cond()) {
1482         case lir_cond_equal:        acond = Assembler::equal;       break;
1483         case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
1484         case lir_cond_less:         acond = Assembler::less;        break;
1485         case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
1486         case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
1487         case lir_cond_greater:      acond = Assembler::greater;     break;
1488         case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
1489         case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
1490         default:                         ShouldNotReachHere();
1491       }
1492     }
1493     __ jcc(acond,*(op->label()));
1494   }
1495 }
1496 
1497 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1498   LIR_Opr src  = op->in_opr();
1499   LIR_Opr dest = op->result_opr();
1500 
1501   switch (op->bytecode()) {
1502     case Bytecodes::_i2l:
1503 #ifdef _LP64
1504       __ movl2ptr(dest->as_register_lo(), src->as_register());
1505 #else
1506       move_regs(src->as_register(), dest->as_register_lo());
1507       move_regs(src->as_register(), dest->as_register_hi());
1508       __ sarl(dest->as_register_hi(), 31);
1509 #endif // LP64
1510       break;
1511 
1512     case Bytecodes::_l2i:
1513 #ifdef _LP64
1514       __ movl(dest->as_register(), src->as_register_lo());
1515 #else
1516       move_regs(src->as_register_lo(), dest->as_register());
1517 #endif
1518       break;
1519 
1520     case Bytecodes::_i2b:
1521       move_regs(src->as_register(), dest->as_register());
1522       __ sign_extend_byte(dest->as_register());
1523       break;
1524 
1525     case Bytecodes::_i2c:
1526       move_regs(src->as_register(), dest->as_register());
1527       __ andl(dest->as_register(), 0xFFFF);
1528       break;
1529 
1530     case Bytecodes::_i2s:
1531       move_regs(src->as_register(), dest->as_register());
1532       __ sign_extend_short(dest->as_register());
1533       break;
1534 
1535 
1536     case Bytecodes::_f2d:
1537     case Bytecodes::_d2f:
1538       if (dest->is_single_xmm()) {
1539         __ cvtsd2ss(dest->as_xmm_float_reg(), src->as_xmm_double_reg());
1540       } else if (dest->is_double_xmm()) {
1541         __ cvtss2sd(dest->as_xmm_double_reg(), src->as_xmm_float_reg());
1542       } else {
1543         assert(src->fpu() == dest->fpu(), "register must be equal");
1544         // do nothing (float result is rounded later through spilling)
1545       }
1546       break;
1547 
1548     case Bytecodes::_i2f:
1549     case Bytecodes::_i2d:
1550       if (dest->is_single_xmm()) {
1551         __ cvtsi2ssl(dest->as_xmm_float_reg(), src->as_register());
1552       } else if (dest->is_double_xmm()) {
1553         __ cvtsi2sdl(dest->as_xmm_double_reg(), src->as_register());
1554       } else {
1555         assert(dest->fpu() == 0, "result must be on TOS");
1556         __ movl(Address(rsp, 0), src->as_register());
1557         __ fild_s(Address(rsp, 0));
1558       }
1559       break;
1560 
1561     case Bytecodes::_f2i:
1562     case Bytecodes::_d2i:
1563       if (src->is_single_xmm()) {
1564         __ cvttss2sil(dest->as_register(), src->as_xmm_float_reg());
1565       } else if (src->is_double_xmm()) {
1566         __ cvttsd2sil(dest->as_register(), src->as_xmm_double_reg());
1567       } else {
1568         assert(src->fpu() == 0, "input must be on TOS");
1569         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_trunc()));
1570         __ fist_s(Address(rsp, 0));
1571         __ movl(dest->as_register(), Address(rsp, 0));
1572         __ fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_std()));
1573       }
1574 
1575       // IA32 conversion instructions do not match JLS for overflow, underflow and NaN -> fixup in stub
1576       assert(op->stub() != NULL, "stub required");
1577       __ cmpl(dest->as_register(), 0x80000000);
1578       __ jcc(Assembler::equal, *op->stub()->entry());
1579       __ bind(*op->stub()->continuation());
1580       break;
1581 
1582     case Bytecodes::_l2f:
1583     case Bytecodes::_l2d:
1584       assert(!dest->is_xmm_register(), "result in xmm register not supported (no SSE instruction present)");
1585       assert(dest->fpu() == 0, "result must be on TOS");
1586 
1587       __ movptr(Address(rsp, 0),            src->as_register_lo());
1588       NOT_LP64(__ movl(Address(rsp, BytesPerWord), src->as_register_hi()));
1589       __ fild_d(Address(rsp, 0));
1590       // float result is rounded later through spilling
1591       break;
1592 
1593     case Bytecodes::_f2l:
1594     case Bytecodes::_d2l:
1595       assert(!src->is_xmm_register(), "input in xmm register not supported (no SSE instruction present)");
1596       assert(src->fpu() == 0, "input must be on TOS");
1597       assert(dest == FrameMap::long0_opr, "runtime stub places result in these registers");
1598 
1599       // instruction sequence too long to inline it here
1600       {
1601         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::fpu2long_stub_id)));
1602       }
1603       break;
1604 
1605     default: ShouldNotReachHere();
1606   }
1607 }
1608 
1609 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1610   if (op->init_check()) {
1611     add_debug_info_for_null_check_here(op->stub()->info());
1612     __ cmpb(Address(op->klass()->as_register(),
1613                     InstanceKlass::init_state_offset()),
1614                     InstanceKlass::fully_initialized);
1615     __ jcc(Assembler::notEqual, *op->stub()->entry());
1616   }
1617   __ allocate_object(op->obj()->as_register(),
1618                      op->tmp1()->as_register(),
1619                      op->tmp2()->as_register(),
1620                      op->header_size(),
1621                      op->object_size(),
1622                      op->klass()->as_register(),
1623                      *op->stub()->entry());
1624   __ bind(*op->stub()->continuation());
1625 }
1626 
1627 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1628   Register len =  op->len()->as_register();
1629   LP64_ONLY( __ movslq(len, len); )
1630 
1631   if (UseSlowPath || op->type() == T_VALUETYPE ||
1632       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1633       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1634     __ jmp(*op->stub()->entry());
1635   } else {
1636     Register tmp1 = op->tmp1()->as_register();
1637     Register tmp2 = op->tmp2()->as_register();
1638     Register tmp3 = op->tmp3()->as_register();
1639     if (len == tmp1) {
1640       tmp1 = tmp3;
1641     } else if (len == tmp2) {
1642       tmp2 = tmp3;
1643     } else if (len == tmp3) {
1644       // everything is ok
1645     } else {
1646       __ mov(tmp3, len);
1647     }
1648     __ allocate_array(op->obj()->as_register(),
1649                       len,
1650                       tmp1,
1651                       tmp2,
1652                       arrayOopDesc::header_size(op->type()),
1653                       array_element_size(op->type()),
1654                       op->klass()->as_register(),
1655                       *op->stub()->entry());
1656   }
1657   __ bind(*op->stub()->continuation());
1658 }
1659 
1660 void LIR_Assembler::type_profile_helper(Register mdo,
1661                                         ciMethodData *md, ciProfileData *data,
1662                                         Register recv, Label* update_done) {
1663   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1664     Label next_test;
1665     // See if the receiver is receiver[n].
1666     __ cmpptr(recv, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1667     __ jccb(Assembler::notEqual, next_test);
1668     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1669     __ addptr(data_addr, DataLayout::counter_increment);
1670     __ jmp(*update_done);
1671     __ bind(next_test);
1672   }
1673 
1674   // Didn't find receiver; find next empty slot and fill it in
1675   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1676     Label next_test;
1677     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)));
1678     __ cmpptr(recv_addr, (intptr_t)NULL_WORD);
1679     __ jccb(Assembler::notEqual, next_test);
1680     __ movptr(recv_addr, recv);
1681     __ movptr(Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))), DataLayout::counter_increment);
1682     __ jmp(*update_done);
1683     __ bind(next_test);
1684   }
1685 }
1686 
1687 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1688   // we always need a stub for the failure case.
1689   CodeStub* stub = op->stub();
1690   Register obj = op->object()->as_register();
1691   Register k_RInfo = op->tmp1()->as_register();
1692   Register klass_RInfo = op->tmp2()->as_register();
1693   Register dst = op->result_opr()->as_register();
1694   ciKlass* k = op->klass();
1695   Register Rtmp1 = noreg;
1696 
1697   // check if it needs to be profiled
1698   ciMethodData* md = NULL;
1699   ciProfileData* data = NULL;
1700 
1701   if (op->should_profile()) {
1702     ciMethod* method = op->profiled_method();
1703     assert(method != NULL, "Should have method");
1704     int bci = op->profiled_bci();
1705     md = method->method_data_or_null();
1706     assert(md != NULL, "Sanity");
1707     data = md->bci_to_data(bci);
1708     assert(data != NULL,                "need data for type check");
1709     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1710   }
1711   Label profile_cast_success, profile_cast_failure;
1712   Label *success_target = op->should_profile() ? &profile_cast_success : success;
1713   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1714 
1715   if (obj == k_RInfo) {
1716     k_RInfo = dst;
1717   } else if (obj == klass_RInfo) {
1718     klass_RInfo = dst;
1719   }
1720   if (k->is_loaded() && !UseCompressedClassPointers) {
1721     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1722   } else {
1723     Rtmp1 = op->tmp3()->as_register();
1724     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1725   }
1726 
1727   assert_different_registers(obj, k_RInfo, klass_RInfo);
1728 
1729   if (op->need_null_check()) {
1730     __ cmpptr(obj, (int32_t)NULL_WORD);
1731     if (op->should_profile()) {
1732       Label not_null;
1733       __ jccb(Assembler::notEqual, not_null);
1734       // Object is null; update MDO and exit
1735       Register mdo  = klass_RInfo;
1736       __ mov_metadata(mdo, md->constant_encoding());
1737       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1738       int header_bits = BitData::null_seen_byte_constant();
1739       __ orb(data_addr, header_bits);
1740       __ jmp(*obj_is_null);
1741       __ bind(not_null);
1742     } else {
1743       __ jcc(Assembler::equal, *obj_is_null);
1744     }
1745   }
1746 
1747   if (!k->is_loaded()) {
1748     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1749   } else {
1750 #ifdef _LP64
1751     __ mov_metadata(k_RInfo, k->constant_encoding());
1752 #endif // _LP64
1753   }
1754   __ verify_oop(obj);
1755 
1756   if (op->fast_check()) {
1757     // get object class
1758     // not a safepoint as obj null check happens earlier
1759 #ifdef _LP64
1760     if (UseCompressedClassPointers) {
1761       __ load_klass(Rtmp1, obj);
1762       __ cmpptr(k_RInfo, Rtmp1);
1763     } else {
1764       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1765     }
1766 #else
1767     if (k->is_loaded()) {
1768       __ cmpklass(Address(obj, oopDesc::klass_offset_in_bytes()), k->constant_encoding());
1769     } else {
1770       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1771     }
1772 #endif
1773     __ jcc(Assembler::notEqual, *failure_target);
1774     // successful cast, fall through to profile or jump
1775   } else {
1776     // get object class
1777     // not a safepoint as obj null check happens earlier
1778     __ load_klass(klass_RInfo, obj);
1779     if (k->is_loaded()) {
1780       // See if we get an immediate positive hit
1781 #ifdef _LP64
1782       __ cmpptr(k_RInfo, Address(klass_RInfo, k->super_check_offset()));
1783 #else
1784       __ cmpklass(Address(klass_RInfo, k->super_check_offset()), k->constant_encoding());
1785 #endif // _LP64
1786       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1787         __ jcc(Assembler::notEqual, *failure_target);
1788         // successful cast, fall through to profile or jump
1789       } else {
1790         // See if we get an immediate positive hit
1791         __ jcc(Assembler::equal, *success_target);
1792         // check for self
1793 #ifdef _LP64
1794         __ cmpptr(klass_RInfo, k_RInfo);
1795 #else
1796         __ cmpklass(klass_RInfo, k->constant_encoding());
1797 #endif // _LP64
1798         __ jcc(Assembler::equal, *success_target);
1799 
1800         __ push(klass_RInfo);
1801 #ifdef _LP64
1802         __ push(k_RInfo);
1803 #else
1804         __ pushklass(k->constant_encoding());
1805 #endif // _LP64
1806         __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1807         __ pop(klass_RInfo);
1808         __ pop(klass_RInfo);
1809         // result is a boolean
1810         __ cmpl(klass_RInfo, 0);
1811         __ jcc(Assembler::equal, *failure_target);
1812         // successful cast, fall through to profile or jump
1813       }
1814     } else {
1815       // perform the fast part of the checking logic
1816       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1817       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1818       __ push(klass_RInfo);
1819       __ push(k_RInfo);
1820       __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1821       __ pop(klass_RInfo);
1822       __ pop(k_RInfo);
1823       // result is a boolean
1824       __ cmpl(k_RInfo, 0);
1825       __ jcc(Assembler::equal, *failure_target);
1826       // successful cast, fall through to profile or jump
1827     }
1828   }
1829   if (op->should_profile()) {
1830     Register mdo  = klass_RInfo, recv = k_RInfo;
1831     __ bind(profile_cast_success);
1832     __ mov_metadata(mdo, md->constant_encoding());
1833     __ load_klass(recv, obj);
1834     type_profile_helper(mdo, md, data, recv, success);
1835     __ jmp(*success);
1836 
1837     __ bind(profile_cast_failure);
1838     __ mov_metadata(mdo, md->constant_encoding());
1839     Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1840     __ subptr(counter_addr, DataLayout::counter_increment);
1841     __ jmp(*failure);
1842   }
1843   __ jmp(*success);
1844 }
1845 
1846 
1847 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1848   LIR_Code code = op->code();
1849   if (code == lir_store_check) {
1850     Register value = op->object()->as_register();
1851     Register array = op->array()->as_register();
1852     Register k_RInfo = op->tmp1()->as_register();
1853     Register klass_RInfo = op->tmp2()->as_register();
1854     Register Rtmp1 = op->tmp3()->as_register();
1855 
1856     CodeStub* stub = op->stub();
1857 
1858     // check if it needs to be profiled
1859     ciMethodData* md = NULL;
1860     ciProfileData* data = NULL;
1861 
1862     if (op->should_profile()) {
1863       ciMethod* method = op->profiled_method();
1864       assert(method != NULL, "Should have method");
1865       int bci = op->profiled_bci();
1866       md = method->method_data_or_null();
1867       assert(md != NULL, "Sanity");
1868       data = md->bci_to_data(bci);
1869       assert(data != NULL,                "need data for type check");
1870       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1871     }
1872     Label profile_cast_success, profile_cast_failure, done;
1873     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1874     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1875 
1876     __ cmpptr(value, (int32_t)NULL_WORD);
1877     if (op->should_profile()) {
1878       Label not_null;
1879       __ jccb(Assembler::notEqual, not_null);
1880       // Object is null; update MDO and exit
1881       Register mdo  = klass_RInfo;
1882       __ mov_metadata(mdo, md->constant_encoding());
1883       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1884       int header_bits = BitData::null_seen_byte_constant();
1885       __ orb(data_addr, header_bits);
1886       __ jmp(done);
1887       __ bind(not_null);
1888     } else {
1889       __ jcc(Assembler::equal, done);
1890     }
1891 
1892     add_debug_info_for_null_check_here(op->info_for_exception());
1893     __ load_klass(k_RInfo, array);
1894     __ load_klass(klass_RInfo, value);
1895 
1896     // get instance klass (it's already uncompressed)
1897     __ movptr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1898     // perform the fast part of the checking logic
1899     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1900     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1901     __ push(klass_RInfo);
1902     __ push(k_RInfo);
1903     __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1904     __ pop(klass_RInfo);
1905     __ pop(k_RInfo);
1906     // result is a boolean
1907     __ cmpl(k_RInfo, 0);
1908     __ jcc(Assembler::equal, *failure_target);
1909     // fall through to the success case
1910 
1911     if (op->should_profile()) {
1912       Register mdo  = klass_RInfo, recv = k_RInfo;
1913       __ bind(profile_cast_success);
1914       __ mov_metadata(mdo, md->constant_encoding());
1915       __ load_klass(recv, value);
1916       type_profile_helper(mdo, md, data, recv, &done);
1917       __ jmpb(done);
1918 
1919       __ bind(profile_cast_failure);
1920       __ mov_metadata(mdo, md->constant_encoding());
1921       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1922       __ subptr(counter_addr, DataLayout::counter_increment);
1923       __ jmp(*stub->entry());
1924     }
1925 
1926     __ bind(done);
1927   } else
1928     if (code == lir_checkcast) {
1929       Register obj = op->object()->as_register();
1930       Register dst = op->result_opr()->as_register();
1931       Label success;
1932       emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1933       __ bind(success);
1934       if (dst != obj) {
1935         __ mov(dst, obj);
1936       }
1937     } else
1938       if (code == lir_instanceof) {
1939         Register obj = op->object()->as_register();
1940         Register dst = op->result_opr()->as_register();
1941         Label success, failure, done;
1942         emit_typecheck_helper(op, &success, &failure, &failure);
1943         __ bind(failure);
1944         __ xorptr(dst, dst);
1945         __ jmpb(done);
1946         __ bind(success);
1947         __ movptr(dst, 1);
1948         __ bind(done);
1949       } else {
1950         ShouldNotReachHere();
1951       }
1952 
1953 }
1954 
1955 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1956   // We are loading/storing an array that *may* be a flattened array (the declared type
1957   // Object[], interface[], or VT?[]). If this array is flattened, take slow path.
1958 
1959   __ load_storage_props(op->tmp()->as_register(), op->array()->as_register());
1960   __ testb(op->tmp()->as_register(), ArrayStorageProperties::flattened_value);
1961   __ jcc(Assembler::notZero, *op->stub()->entry());
1962   if (!op->value()->is_illegal()) {
1963     // We are storing into the array.
1964     Label skip;
1965     __ testb(op->tmp()->as_register(), ArrayStorageProperties::null_free_value);
1966     __ jcc(Assembler::zero, skip);
1967     // The array is not flattened, but it is null_free. If we are storing
1968     // a null, take the slow path (which will throw NPE).
1969     __ cmpptr(op->value()->as_register(), (int32_t)NULL_WORD);
1970     __ jcc(Assembler::zero, *op->stub()->entry());
1971     __ bind(skip);
1972   }
1973 }
1974 
1975 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1976   // This is called when we use aastore into a an array declared as "[LVT;",
1977   // where we know VT is not flattenable (due to ValueArrayElemMaxFlatOops, etc).
1978   // However, we need to do a NULL check if the actual array is a "[QVT;".
1979 
1980   __ load_storage_props(op->tmp()->as_register(), op->array()->as_register());
1981   __ testb(op->tmp()->as_register(), ArrayStorageProperties::null_free_value);
1982 }
1983 
1984 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1985   Label L_oops_equal;
1986   Label L_oops_not_equal;
1987   Label L_end;
1988 
1989   Register left  = op->left()->as_register();
1990   Register right = op->right()->as_register();
1991 
1992   __ cmpptr(left, right);
1993   __ jcc(Assembler::equal, L_oops_equal);
1994 
1995   // (1) Null check -- if one of the operands is null, the other must not be null (because
1996   //     the two references are not equal), so they are not substitutable,
1997   //     FIXME: do null check only if the operand is nullable
1998   {
1999     __ cmpptr(left, (int32_t)NULL_WORD);
2000     __ jcc(Assembler::equal, L_oops_not_equal);
2001 
2002     __ cmpptr(right, (int32_t)NULL_WORD);
2003     __ jcc(Assembler::equal, L_oops_not_equal);
2004   }
2005 
2006   ciKlass* left_klass = op->left_klass();
2007   ciKlass* right_klass = op->right_klass();
2008 
2009   // (2) Value object check -- if either of the operands is not a value object,
2010   //     they are not substitutable. We do this only if we are not sure that the
2011   //     operands are value objects
2012   if ((left_klass == NULL || right_klass == NULL) ||// The klass is still unloaded, or came from a Phi node.
2013       !left_klass->is_valuetype() || !right_klass->is_valuetype()) {
2014     Register tmp1  = op->tmp1()->as_register();
2015     __ movptr(tmp1, (intptr_t)markOopDesc::always_locked_pattern);
2016     __ andl(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
2017     __ andl(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
2018     __ cmpptr(tmp1, (intptr_t)markOopDesc::always_locked_pattern);
2019     __ jcc(Assembler::notEqual, L_oops_not_equal);
2020   }
2021 
2022   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
2023   if (left_klass != NULL && left_klass->is_valuetype() && left_klass == right_klass) {
2024     // No need to load klass -- the operands are statically known to be the same value klass.
2025     __ jmp(*op->stub()->entry());
2026   } else {
2027     Register left_klass_op = op->left_klass_op()->as_register();
2028     Register right_klass_op = op->right_klass_op()->as_register();
2029 
2030     if (UseCompressedOops) {
2031       __ movl(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
2032       __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2033       __ cmpl(left_klass_op, right_klass_op);
2034     } else {
2035       __ movptr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
2036       __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
2037       __ cmpptr(left_klass_op, right_klass_op);
2038     }
2039 
2040     __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
2041     // fall through to L_oops_not_equal
2042   }
2043 
2044   __ bind(L_oops_not_equal);
2045   move(op->not_equal_result(), op->result_opr());
2046   __ jmp(L_end);
2047 
2048   __ bind(L_oops_equal);
2049   move(op->equal_result(), op->result_opr());
2050   __ jmp(L_end);
2051 
2052   // We've returned from the stub. RAX contains 0x0 IFF the two
2053   // operands are not substitutable. (Don't compare against 0x1 in case the
2054   // C compiler is naughty)
2055   __ bind(*op->stub()->continuation());
2056   __ cmpl(rax, 0);
2057   __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
2058   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
2059   // fall-through
2060   __ bind(L_end);
2061 }
2062 
2063 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2064   if (LP64_ONLY(false &&) op->code() == lir_cas_long && VM_Version::supports_cx8()) {
2065     assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
2066     assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
2067     assert(op->new_value()->as_register_lo() == rbx, "wrong register");
2068     assert(op->new_value()->as_register_hi() == rcx, "wrong register");
2069     Register addr = op->addr()->as_register();
2070     __ lock();
2071     NOT_LP64(__ cmpxchg8(Address(addr, 0)));
2072 
2073   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
2074     NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
2075     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2076     Register newval = op->new_value()->as_register();
2077     Register cmpval = op->cmp_value()->as_register();
2078     assert(cmpval == rax, "wrong register");
2079     assert(newval != NULL, "new val must be register");
2080     assert(cmpval != newval, "cmp and new values must be in different registers");
2081     assert(cmpval != addr, "cmp and addr must be in different registers");
2082     assert(newval != addr, "new value and addr must be in different registers");
2083 
2084     if ( op->code() == lir_cas_obj) {
2085 #ifdef _LP64
2086       if (UseCompressedOops) {
2087         __ encode_heap_oop(cmpval);
2088         __ mov(rscratch1, newval);
2089         __ encode_heap_oop(rscratch1);
2090         __ lock();
2091         // cmpval (rax) is implicitly used by this instruction
2092         __ cmpxchgl(rscratch1, Address(addr, 0));
2093       } else
2094 #endif
2095       {
2096         __ lock();
2097         __ cmpxchgptr(newval, Address(addr, 0));
2098       }
2099     } else {
2100       assert(op->code() == lir_cas_int, "lir_cas_int expected");
2101       __ lock();
2102       __ cmpxchgl(newval, Address(addr, 0));
2103     }
2104 #ifdef _LP64
2105   } else if (op->code() == lir_cas_long) {
2106     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
2107     Register newval = op->new_value()->as_register_lo();
2108     Register cmpval = op->cmp_value()->as_register_lo();
2109     assert(cmpval == rax, "wrong register");
2110     assert(newval != NULL, "new val must be register");
2111     assert(cmpval != newval, "cmp and new values must be in different registers");
2112     assert(cmpval != addr, "cmp and addr must be in different registers");
2113     assert(newval != addr, "new value and addr must be in different registers");
2114     __ lock();
2115     __ cmpxchgq(newval, Address(addr, 0));
2116 #endif // _LP64
2117   } else {
2118     Unimplemented();
2119   }
2120 }
2121 
2122 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
2123   assert(dst->is_cpu_register(), "must be");
2124   assert(dst->type() == src->type(), "must be");
2125 
2126   if (src->is_cpu_register()) {
2127     reg2reg(src, dst);
2128   } else if (src->is_stack()) {
2129     stack2reg(src, dst, dst->type());
2130   } else if (src->is_constant()) {
2131     const2reg(src, dst, lir_patch_none, NULL);
2132   } else {
2133     ShouldNotReachHere();
2134   }
2135 }
2136 
2137 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
2138   Assembler::Condition acond, ncond;
2139   switch (condition) {
2140     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
2141     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
2142     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
2143     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
2144     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
2145     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
2146     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
2147     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
2148     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
2149                                 ShouldNotReachHere();
2150   }
2151 
2152   if (opr1->is_cpu_register()) {
2153     reg2reg(opr1, result);
2154   } else if (opr1->is_stack()) {
2155     stack2reg(opr1, result, result->type());
2156   } else if (opr1->is_constant()) {
2157     const2reg(opr1, result, lir_patch_none, NULL);
2158   } else {
2159     ShouldNotReachHere();
2160   }
2161 
2162   if (VM_Version::supports_cmov() && !opr2->is_constant()) {
2163     // optimized version that does not require a branch
2164     if (opr2->is_single_cpu()) {
2165       assert(opr2->cpu_regnr() != result->cpu_regnr(), "opr2 already overwritten by previous move");
2166       __ cmov(ncond, result->as_register(), opr2->as_register());
2167     } else if (opr2->is_double_cpu()) {
2168       assert(opr2->cpu_regnrLo() != result->cpu_regnrLo() && opr2->cpu_regnrLo() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2169       assert(opr2->cpu_regnrHi() != result->cpu_regnrLo() && opr2->cpu_regnrHi() != result->cpu_regnrHi(), "opr2 already overwritten by previous move");
2170       __ cmovptr(ncond, result->as_register_lo(), opr2->as_register_lo());
2171       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), opr2->as_register_hi());)
2172     } else if (opr2->is_single_stack()) {
2173       __ cmovl(ncond, result->as_register(), frame_map()->address_for_slot(opr2->single_stack_ix()));
2174     } else if (opr2->is_double_stack()) {
2175       __ cmovptr(ncond, result->as_register_lo(), frame_map()->address_for_slot(opr2->double_stack_ix(), lo_word_offset_in_bytes));
2176       NOT_LP64(__ cmovptr(ncond, result->as_register_hi(), frame_map()->address_for_slot(opr2->double_stack_ix(), hi_word_offset_in_bytes));)
2177     } else {
2178       ShouldNotReachHere();
2179     }
2180 
2181   } else {
2182     Label skip;
2183     __ jcc (acond, skip);
2184     if (opr2->is_cpu_register()) {
2185       reg2reg(opr2, result);
2186     } else if (opr2->is_stack()) {
2187       stack2reg(opr2, result, result->type());
2188     } else if (opr2->is_constant()) {
2189       const2reg(opr2, result, lir_patch_none, NULL);
2190     } else {
2191       ShouldNotReachHere();
2192     }
2193     __ bind(skip);
2194   }
2195 }
2196 
2197 
2198 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
2199   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
2200 
2201   if (left->is_single_cpu()) {
2202     assert(left == dest, "left and dest must be equal");
2203     Register lreg = left->as_register();
2204 
2205     if (right->is_single_cpu()) {
2206       // cpu register - cpu register
2207       Register rreg = right->as_register();
2208       switch (code) {
2209         case lir_add: __ addl (lreg, rreg); break;
2210         case lir_sub: __ subl (lreg, rreg); break;
2211         case lir_mul: __ imull(lreg, rreg); break;
2212         default:      ShouldNotReachHere();
2213       }
2214 
2215     } else if (right->is_stack()) {
2216       // cpu register - stack
2217       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2218       switch (code) {
2219         case lir_add: __ addl(lreg, raddr); break;
2220         case lir_sub: __ subl(lreg, raddr); break;
2221         default:      ShouldNotReachHere();
2222       }
2223 
2224     } else if (right->is_constant()) {
2225       // cpu register - constant
2226       jint c = right->as_constant_ptr()->as_jint();
2227       switch (code) {
2228         case lir_add: {
2229           __ incrementl(lreg, c);
2230           break;
2231         }
2232         case lir_sub: {
2233           __ decrementl(lreg, c);
2234           break;
2235         }
2236         default: ShouldNotReachHere();
2237       }
2238 
2239     } else {
2240       ShouldNotReachHere();
2241     }
2242 
2243   } else if (left->is_double_cpu()) {
2244     assert(left == dest, "left and dest must be equal");
2245     Register lreg_lo = left->as_register_lo();
2246     Register lreg_hi = left->as_register_hi();
2247 
2248     if (right->is_double_cpu()) {
2249       // cpu register - cpu register
2250       Register rreg_lo = right->as_register_lo();
2251       Register rreg_hi = right->as_register_hi();
2252       NOT_LP64(assert_different_registers(lreg_lo, lreg_hi, rreg_lo, rreg_hi));
2253       LP64_ONLY(assert_different_registers(lreg_lo, rreg_lo));
2254       switch (code) {
2255         case lir_add:
2256           __ addptr(lreg_lo, rreg_lo);
2257           NOT_LP64(__ adcl(lreg_hi, rreg_hi));
2258           break;
2259         case lir_sub:
2260           __ subptr(lreg_lo, rreg_lo);
2261           NOT_LP64(__ sbbl(lreg_hi, rreg_hi));
2262           break;
2263         case lir_mul:
2264 #ifdef _LP64
2265           __ imulq(lreg_lo, rreg_lo);
2266 #else
2267           assert(lreg_lo == rax && lreg_hi == rdx, "must be");
2268           __ imull(lreg_hi, rreg_lo);
2269           __ imull(rreg_hi, lreg_lo);
2270           __ addl (rreg_hi, lreg_hi);
2271           __ mull (rreg_lo);
2272           __ addl (lreg_hi, rreg_hi);
2273 #endif // _LP64
2274           break;
2275         default:
2276           ShouldNotReachHere();
2277       }
2278 
2279     } else if (right->is_constant()) {
2280       // cpu register - constant
2281 #ifdef _LP64
2282       jlong c = right->as_constant_ptr()->as_jlong_bits();
2283       __ movptr(r10, (intptr_t) c);
2284       switch (code) {
2285         case lir_add:
2286           __ addptr(lreg_lo, r10);
2287           break;
2288         case lir_sub:
2289           __ subptr(lreg_lo, r10);
2290           break;
2291         default:
2292           ShouldNotReachHere();
2293       }
2294 #else
2295       jint c_lo = right->as_constant_ptr()->as_jint_lo();
2296       jint c_hi = right->as_constant_ptr()->as_jint_hi();
2297       switch (code) {
2298         case lir_add:
2299           __ addptr(lreg_lo, c_lo);
2300           __ adcl(lreg_hi, c_hi);
2301           break;
2302         case lir_sub:
2303           __ subptr(lreg_lo, c_lo);
2304           __ sbbl(lreg_hi, c_hi);
2305           break;
2306         default:
2307           ShouldNotReachHere();
2308       }
2309 #endif // _LP64
2310 
2311     } else {
2312       ShouldNotReachHere();
2313     }
2314 
2315   } else if (left->is_single_xmm()) {
2316     assert(left == dest, "left and dest must be equal");
2317     XMMRegister lreg = left->as_xmm_float_reg();
2318 
2319     if (right->is_single_xmm()) {
2320       XMMRegister rreg = right->as_xmm_float_reg();
2321       switch (code) {
2322         case lir_add: __ addss(lreg, rreg);  break;
2323         case lir_sub: __ subss(lreg, rreg);  break;
2324         case lir_mul_strictfp: // fall through
2325         case lir_mul: __ mulss(lreg, rreg);  break;
2326         case lir_div_strictfp: // fall through
2327         case lir_div: __ divss(lreg, rreg);  break;
2328         default: ShouldNotReachHere();
2329       }
2330     } else {
2331       Address raddr;
2332       if (right->is_single_stack()) {
2333         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2334       } else if (right->is_constant()) {
2335         // hack for now
2336         raddr = __ as_Address(InternalAddress(float_constant(right->as_jfloat())));
2337       } else {
2338         ShouldNotReachHere();
2339       }
2340       switch (code) {
2341         case lir_add: __ addss(lreg, raddr);  break;
2342         case lir_sub: __ subss(lreg, raddr);  break;
2343         case lir_mul_strictfp: // fall through
2344         case lir_mul: __ mulss(lreg, raddr);  break;
2345         case lir_div_strictfp: // fall through
2346         case lir_div: __ divss(lreg, raddr);  break;
2347         default: ShouldNotReachHere();
2348       }
2349     }
2350 
2351   } else if (left->is_double_xmm()) {
2352     assert(left == dest, "left and dest must be equal");
2353 
2354     XMMRegister lreg = left->as_xmm_double_reg();
2355     if (right->is_double_xmm()) {
2356       XMMRegister rreg = right->as_xmm_double_reg();
2357       switch (code) {
2358         case lir_add: __ addsd(lreg, rreg);  break;
2359         case lir_sub: __ subsd(lreg, rreg);  break;
2360         case lir_mul_strictfp: // fall through
2361         case lir_mul: __ mulsd(lreg, rreg);  break;
2362         case lir_div_strictfp: // fall through
2363         case lir_div: __ divsd(lreg, rreg);  break;
2364         default: ShouldNotReachHere();
2365       }
2366     } else {
2367       Address raddr;
2368       if (right->is_double_stack()) {
2369         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2370       } else if (right->is_constant()) {
2371         // hack for now
2372         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2373       } else {
2374         ShouldNotReachHere();
2375       }
2376       switch (code) {
2377         case lir_add: __ addsd(lreg, raddr);  break;
2378         case lir_sub: __ subsd(lreg, raddr);  break;
2379         case lir_mul_strictfp: // fall through
2380         case lir_mul: __ mulsd(lreg, raddr);  break;
2381         case lir_div_strictfp: // fall through
2382         case lir_div: __ divsd(lreg, raddr);  break;
2383         default: ShouldNotReachHere();
2384       }
2385     }
2386 
2387   } else if (left->is_single_fpu()) {
2388     assert(dest->is_single_fpu(),  "fpu stack allocation required");
2389 
2390     if (right->is_single_fpu()) {
2391       arith_fpu_implementation(code, left->fpu_regnr(), right->fpu_regnr(), dest->fpu_regnr(), pop_fpu_stack);
2392 
2393     } else {
2394       assert(left->fpu_regnr() == 0, "left must be on TOS");
2395       assert(dest->fpu_regnr() == 0, "dest must be on TOS");
2396 
2397       Address raddr;
2398       if (right->is_single_stack()) {
2399         raddr = frame_map()->address_for_slot(right->single_stack_ix());
2400       } else if (right->is_constant()) {
2401         address const_addr = float_constant(right->as_jfloat());
2402         assert(const_addr != NULL, "incorrect float/double constant maintainance");
2403         // hack for now
2404         raddr = __ as_Address(InternalAddress(const_addr));
2405       } else {
2406         ShouldNotReachHere();
2407       }
2408 
2409       switch (code) {
2410         case lir_add: __ fadd_s(raddr); break;
2411         case lir_sub: __ fsub_s(raddr); break;
2412         case lir_mul_strictfp: // fall through
2413         case lir_mul: __ fmul_s(raddr); break;
2414         case lir_div_strictfp: // fall through
2415         case lir_div: __ fdiv_s(raddr); break;
2416         default:      ShouldNotReachHere();
2417       }
2418     }
2419 
2420   } else if (left->is_double_fpu()) {
2421     assert(dest->is_double_fpu(),  "fpu stack allocation required");
2422 
2423     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2424       // Double values require special handling for strictfp mul/div on x86
2425       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias1()));
2426       __ fmulp(left->fpu_regnrLo() + 1);
2427     }
2428 
2429     if (right->is_double_fpu()) {
2430       arith_fpu_implementation(code, left->fpu_regnrLo(), right->fpu_regnrLo(), dest->fpu_regnrLo(), pop_fpu_stack);
2431 
2432     } else {
2433       assert(left->fpu_regnrLo() == 0, "left must be on TOS");
2434       assert(dest->fpu_regnrLo() == 0, "dest must be on TOS");
2435 
2436       Address raddr;
2437       if (right->is_double_stack()) {
2438         raddr = frame_map()->address_for_slot(right->double_stack_ix());
2439       } else if (right->is_constant()) {
2440         // hack for now
2441         raddr = __ as_Address(InternalAddress(double_constant(right->as_jdouble())));
2442       } else {
2443         ShouldNotReachHere();
2444       }
2445 
2446       switch (code) {
2447         case lir_add: __ fadd_d(raddr); break;
2448         case lir_sub: __ fsub_d(raddr); break;
2449         case lir_mul_strictfp: // fall through
2450         case lir_mul: __ fmul_d(raddr); break;
2451         case lir_div_strictfp: // fall through
2452         case lir_div: __ fdiv_d(raddr); break;
2453         default: ShouldNotReachHere();
2454       }
2455     }
2456 
2457     if (code == lir_mul_strictfp || code == lir_div_strictfp) {
2458       // Double values require special handling for strictfp mul/div on x86
2459       __ fld_x(ExternalAddress(StubRoutines::addr_fpu_subnormal_bias2()));
2460       __ fmulp(dest->fpu_regnrLo() + 1);
2461     }
2462 
2463   } else if (left->is_single_stack() || left->is_address()) {
2464     assert(left == dest, "left and dest must be equal");
2465 
2466     Address laddr;
2467     if (left->is_single_stack()) {
2468       laddr = frame_map()->address_for_slot(left->single_stack_ix());
2469     } else if (left->is_address()) {
2470       laddr = as_Address(left->as_address_ptr());
2471     } else {
2472       ShouldNotReachHere();
2473     }
2474 
2475     if (right->is_single_cpu()) {
2476       Register rreg = right->as_register();
2477       switch (code) {
2478         case lir_add: __ addl(laddr, rreg); break;
2479         case lir_sub: __ subl(laddr, rreg); break;
2480         default:      ShouldNotReachHere();
2481       }
2482     } else if (right->is_constant()) {
2483       jint c = right->as_constant_ptr()->as_jint();
2484       switch (code) {
2485         case lir_add: {
2486           __ incrementl(laddr, c);
2487           break;
2488         }
2489         case lir_sub: {
2490           __ decrementl(laddr, c);
2491           break;
2492         }
2493         default: ShouldNotReachHere();
2494       }
2495     } else {
2496       ShouldNotReachHere();
2497     }
2498 
2499   } else {
2500     ShouldNotReachHere();
2501   }
2502 }
2503 
2504 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) {
2505   assert(pop_fpu_stack  || (left_index     == dest_index || right_index     == dest_index), "invalid LIR");
2506   assert(!pop_fpu_stack || (left_index - 1 == dest_index || right_index - 1 == dest_index), "invalid LIR");
2507   assert(left_index == 0 || right_index == 0, "either must be on top of stack");
2508 
2509   bool left_is_tos = (left_index == 0);
2510   bool dest_is_tos = (dest_index == 0);
2511   int non_tos_index = (left_is_tos ? right_index : left_index);
2512 
2513   switch (code) {
2514     case lir_add:
2515       if (pop_fpu_stack)       __ faddp(non_tos_index);
2516       else if (dest_is_tos)    __ fadd (non_tos_index);
2517       else                     __ fadda(non_tos_index);
2518       break;
2519 
2520     case lir_sub:
2521       if (left_is_tos) {
2522         if (pop_fpu_stack)     __ fsubrp(non_tos_index);
2523         else if (dest_is_tos)  __ fsub  (non_tos_index);
2524         else                   __ fsubra(non_tos_index);
2525       } else {
2526         if (pop_fpu_stack)     __ fsubp (non_tos_index);
2527         else if (dest_is_tos)  __ fsubr (non_tos_index);
2528         else                   __ fsuba (non_tos_index);
2529       }
2530       break;
2531 
2532     case lir_mul_strictfp: // fall through
2533     case lir_mul:
2534       if (pop_fpu_stack)       __ fmulp(non_tos_index);
2535       else if (dest_is_tos)    __ fmul (non_tos_index);
2536       else                     __ fmula(non_tos_index);
2537       break;
2538 
2539     case lir_div_strictfp: // fall through
2540     case lir_div:
2541       if (left_is_tos) {
2542         if (pop_fpu_stack)     __ fdivrp(non_tos_index);
2543         else if (dest_is_tos)  __ fdiv  (non_tos_index);
2544         else                   __ fdivra(non_tos_index);
2545       } else {
2546         if (pop_fpu_stack)     __ fdivp (non_tos_index);
2547         else if (dest_is_tos)  __ fdivr (non_tos_index);
2548         else                   __ fdiva (non_tos_index);
2549       }
2550       break;
2551 
2552     case lir_rem:
2553       assert(left_is_tos && dest_is_tos && right_index == 1, "must be guaranteed by FPU stack allocation");
2554       __ fremr(noreg);
2555       break;
2556 
2557     default:
2558       ShouldNotReachHere();
2559   }
2560 }
2561 
2562 
2563 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
2564   if (value->is_double_xmm()) {
2565     switch(code) {
2566       case lir_abs :
2567         {
2568 #ifdef _LP64
2569           if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
2570             assert(tmp->is_valid(), "need temporary");
2571             __ vpandn(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), value->as_xmm_double_reg(), 2);
2572           } else
2573 #endif
2574           {
2575             if (dest->as_xmm_double_reg() != value->as_xmm_double_reg()) {
2576               __ movdbl(dest->as_xmm_double_reg(), value->as_xmm_double_reg());
2577             }
2578             assert(!tmp->is_valid(), "do not need temporary");
2579             __ andpd(dest->as_xmm_double_reg(),
2580                      ExternalAddress((address)double_signmask_pool));
2581           }
2582         }
2583         break;
2584 
2585       case lir_sqrt: __ sqrtsd(dest->as_xmm_double_reg(), value->as_xmm_double_reg()); break;
2586       // all other intrinsics are not available in the SSE instruction set, so FPU is used
2587       default      : ShouldNotReachHere();
2588     }
2589 
2590   } else if (value->is_double_fpu()) {
2591     assert(value->fpu_regnrLo() == 0 && dest->fpu_regnrLo() == 0, "both must be on TOS");
2592     switch(code) {
2593       case lir_abs   : __ fabs() ; break;
2594       case lir_sqrt  : __ fsqrt(); break;
2595       default      : ShouldNotReachHere();
2596     }
2597   } else {
2598     Unimplemented();
2599   }
2600 }
2601 
2602 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2603   // assert(left->destroys_register(), "check");
2604   if (left->is_single_cpu()) {
2605     Register reg = left->as_register();
2606     if (right->is_constant()) {
2607       int val = right->as_constant_ptr()->as_jint();
2608       switch (code) {
2609         case lir_logic_and: __ andl (reg, val); break;
2610         case lir_logic_or:  __ orl  (reg, val); break;
2611         case lir_logic_xor: __ xorl (reg, val); break;
2612         default: ShouldNotReachHere();
2613       }
2614     } else if (right->is_stack()) {
2615       // added support for stack operands
2616       Address raddr = frame_map()->address_for_slot(right->single_stack_ix());
2617       switch (code) {
2618         case lir_logic_and: __ andl (reg, raddr); break;
2619         case lir_logic_or:  __ orl  (reg, raddr); break;
2620         case lir_logic_xor: __ xorl (reg, raddr); break;
2621         default: ShouldNotReachHere();
2622       }
2623     } else {
2624       Register rright = right->as_register();
2625       switch (code) {
2626         case lir_logic_and: __ andptr (reg, rright); break;
2627         case lir_logic_or : __ orptr  (reg, rright); break;
2628         case lir_logic_xor: __ xorptr (reg, rright); break;
2629         default: ShouldNotReachHere();
2630       }
2631     }
2632     move_regs(reg, dst->as_register());
2633   } else {
2634     Register l_lo = left->as_register_lo();
2635     Register l_hi = left->as_register_hi();
2636     if (right->is_constant()) {
2637 #ifdef _LP64
2638       __ mov64(rscratch1, right->as_constant_ptr()->as_jlong());
2639       switch (code) {
2640         case lir_logic_and:
2641           __ andq(l_lo, rscratch1);
2642           break;
2643         case lir_logic_or:
2644           __ orq(l_lo, rscratch1);
2645           break;
2646         case lir_logic_xor:
2647           __ xorq(l_lo, rscratch1);
2648           break;
2649         default: ShouldNotReachHere();
2650       }
2651 #else
2652       int r_lo = right->as_constant_ptr()->as_jint_lo();
2653       int r_hi = right->as_constant_ptr()->as_jint_hi();
2654       switch (code) {
2655         case lir_logic_and:
2656           __ andl(l_lo, r_lo);
2657           __ andl(l_hi, r_hi);
2658           break;
2659         case lir_logic_or:
2660           __ orl(l_lo, r_lo);
2661           __ orl(l_hi, r_hi);
2662           break;
2663         case lir_logic_xor:
2664           __ xorl(l_lo, r_lo);
2665           __ xorl(l_hi, r_hi);
2666           break;
2667         default: ShouldNotReachHere();
2668       }
2669 #endif // _LP64
2670     } else {
2671 #ifdef _LP64
2672       Register r_lo;
2673       if (right->type() == T_OBJECT || right->type() == T_ARRAY || right->type() == T_VALUETYPE) {
2674         r_lo = right->as_register();
2675       } else {
2676         r_lo = right->as_register_lo();
2677       }
2678 #else
2679       Register r_lo = right->as_register_lo();
2680       Register r_hi = right->as_register_hi();
2681       assert(l_lo != r_hi, "overwriting registers");
2682 #endif
2683       switch (code) {
2684         case lir_logic_and:
2685           __ andptr(l_lo, r_lo);
2686           NOT_LP64(__ andptr(l_hi, r_hi);)
2687           break;
2688         case lir_logic_or:
2689           __ orptr(l_lo, r_lo);
2690           NOT_LP64(__ orptr(l_hi, r_hi);)
2691           break;
2692         case lir_logic_xor:
2693           __ xorptr(l_lo, r_lo);
2694           NOT_LP64(__ xorptr(l_hi, r_hi);)
2695           break;
2696         default: ShouldNotReachHere();
2697       }
2698     }
2699 
2700     Register dst_lo = dst->as_register_lo();
2701     Register dst_hi = dst->as_register_hi();
2702 
2703 #ifdef _LP64
2704     move_regs(l_lo, dst_lo);
2705 #else
2706     if (dst_lo == l_hi) {
2707       assert(dst_hi != l_lo, "overwriting registers");
2708       move_regs(l_hi, dst_hi);
2709       move_regs(l_lo, dst_lo);
2710     } else {
2711       assert(dst_lo != l_hi, "overwriting registers");
2712       move_regs(l_lo, dst_lo);
2713       move_regs(l_hi, dst_hi);
2714     }
2715 #endif // _LP64
2716   }
2717 }
2718 
2719 
2720 // we assume that rax, and rdx can be overwritten
2721 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) {
2722 
2723   assert(left->is_single_cpu(),   "left must be register");
2724   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2725   assert(result->is_single_cpu(), "result must be register");
2726 
2727   //  assert(left->destroys_register(), "check");
2728   //  assert(right->destroys_register(), "check");
2729 
2730   Register lreg = left->as_register();
2731   Register dreg = result->as_register();
2732 
2733   if (right->is_constant()) {
2734     jint divisor = right->as_constant_ptr()->as_jint();
2735     assert(divisor > 0 && is_power_of_2(divisor), "must be");
2736     if (code == lir_idiv) {
2737       assert(lreg == rax, "must be rax,");
2738       assert(temp->as_register() == rdx, "tmp register must be rdx");
2739       __ cdql(); // sign extend into rdx:rax
2740       if (divisor == 2) {
2741         __ subl(lreg, rdx);
2742       } else {
2743         __ andl(rdx, divisor - 1);
2744         __ addl(lreg, rdx);
2745       }
2746       __ sarl(lreg, log2_jint(divisor));
2747       move_regs(lreg, dreg);
2748     } else if (code == lir_irem) {
2749       Label done;
2750       __ mov(dreg, lreg);
2751       __ andl(dreg, 0x80000000 | (divisor - 1));
2752       __ jcc(Assembler::positive, done);
2753       __ decrement(dreg);
2754       __ orl(dreg, ~(divisor - 1));
2755       __ increment(dreg);
2756       __ bind(done);
2757     } else {
2758       ShouldNotReachHere();
2759     }
2760   } else {
2761     Register rreg = right->as_register();
2762     assert(lreg == rax, "left register must be rax,");
2763     assert(rreg != rdx, "right register must not be rdx");
2764     assert(temp->as_register() == rdx, "tmp register must be rdx");
2765 
2766     move_regs(lreg, rax);
2767 
2768     int idivl_offset = __ corrected_idivl(rreg);
2769     if (ImplicitDiv0Checks) {
2770       add_debug_info_for_div0(idivl_offset, info);
2771     }
2772     if (code == lir_irem) {
2773       move_regs(rdx, dreg); // result is in rdx
2774     } else {
2775       move_regs(rax, dreg);
2776     }
2777   }
2778 }
2779 
2780 
2781 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2782   if (opr1->is_single_cpu()) {
2783     Register reg1 = opr1->as_register();
2784     if (opr2->is_single_cpu()) {
2785       // cpu register - cpu register
2786       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY || opr1->type() == T_VALUETYPE) {
2787         __ cmpoop(reg1, opr2->as_register());
2788       } else {
2789         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY && opr2->type() != T_VALUETYPE, "cmp int, oop?");
2790         __ cmpl(reg1, opr2->as_register());
2791       }
2792     } else if (opr2->is_stack()) {
2793       // cpu register - stack
2794       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY || opr1->type() == T_VALUETYPE) {
2795         __ cmpoop(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2796       } else {
2797         __ cmpl(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2798       }
2799     } else if (opr2->is_constant()) {
2800       // cpu register - constant
2801       LIR_Const* c = opr2->as_constant_ptr();
2802       if (c->type() == T_INT) {
2803         __ cmpl(reg1, c->as_jint());
2804       } else if (c->type() == T_OBJECT || c->type() == T_ARRAY || c->type() == T_VALUETYPE) {
2805         // In 64bit oops are single register
2806         jobject o = c->as_jobject();
2807         if (o == NULL) {
2808           __ cmpptr(reg1, (int32_t)NULL_WORD);
2809         } else {
2810           __ cmpoop(reg1, o);
2811         }
2812       } else {
2813         fatal("unexpected type: %s", basictype_to_str(c->type()));
2814       }
2815       // cpu register - address
2816     } else if (opr2->is_address()) {
2817       if (op->info() != NULL) {
2818         add_debug_info_for_null_check_here(op->info());
2819       }
2820       __ cmpl(reg1, as_Address(opr2->as_address_ptr()));
2821     } else {
2822       ShouldNotReachHere();
2823     }
2824 
2825   } else if(opr1->is_double_cpu()) {
2826     Register xlo = opr1->as_register_lo();
2827     Register xhi = opr1->as_register_hi();
2828     if (opr2->is_double_cpu()) {
2829 #ifdef _LP64
2830       __ cmpptr(xlo, opr2->as_register_lo());
2831 #else
2832       // cpu register - cpu register
2833       Register ylo = opr2->as_register_lo();
2834       Register yhi = opr2->as_register_hi();
2835       __ subl(xlo, ylo);
2836       __ sbbl(xhi, yhi);
2837       if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
2838         __ orl(xhi, xlo);
2839       }
2840 #endif // _LP64
2841     } else if (opr2->is_constant()) {
2842       // cpu register - constant 0
2843       assert(opr2->as_jlong() == (jlong)0, "only handles zero");
2844 #ifdef _LP64
2845       __ cmpptr(xlo, (int32_t)opr2->as_jlong());
2846 #else
2847       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles equals case");
2848       __ orl(xhi, xlo);
2849 #endif // _LP64
2850     } else {
2851       ShouldNotReachHere();
2852     }
2853 
2854   } else if (opr1->is_single_xmm()) {
2855     XMMRegister reg1 = opr1->as_xmm_float_reg();
2856     if (opr2->is_single_xmm()) {
2857       // xmm register - xmm register
2858       __ ucomiss(reg1, opr2->as_xmm_float_reg());
2859     } else if (opr2->is_stack()) {
2860       // xmm register - stack
2861       __ ucomiss(reg1, frame_map()->address_for_slot(opr2->single_stack_ix()));
2862     } else if (opr2->is_constant()) {
2863       // xmm register - constant
2864       __ ucomiss(reg1, InternalAddress(float_constant(opr2->as_jfloat())));
2865     } else if (opr2->is_address()) {
2866       // xmm register - address
2867       if (op->info() != NULL) {
2868         add_debug_info_for_null_check_here(op->info());
2869       }
2870       __ ucomiss(reg1, as_Address(opr2->as_address_ptr()));
2871     } else {
2872       ShouldNotReachHere();
2873     }
2874 
2875   } else if (opr1->is_double_xmm()) {
2876     XMMRegister reg1 = opr1->as_xmm_double_reg();
2877     if (opr2->is_double_xmm()) {
2878       // xmm register - xmm register
2879       __ ucomisd(reg1, opr2->as_xmm_double_reg());
2880     } else if (opr2->is_stack()) {
2881       // xmm register - stack
2882       __ ucomisd(reg1, frame_map()->address_for_slot(opr2->double_stack_ix()));
2883     } else if (opr2->is_constant()) {
2884       // xmm register - constant
2885       __ ucomisd(reg1, InternalAddress(double_constant(opr2->as_jdouble())));
2886     } else if (opr2->is_address()) {
2887       // xmm register - address
2888       if (op->info() != NULL) {
2889         add_debug_info_for_null_check_here(op->info());
2890       }
2891       __ ucomisd(reg1, as_Address(opr2->pointer()->as_address()));
2892     } else {
2893       ShouldNotReachHere();
2894     }
2895 
2896   } else if(opr1->is_single_fpu() || opr1->is_double_fpu()) {
2897     assert(opr1->is_fpu_register() && opr1->fpu() == 0, "currently left-hand side must be on TOS (relax this restriction)");
2898     assert(opr2->is_fpu_register(), "both must be registers");
2899     __ fcmp(noreg, opr2->fpu(), op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2900 
2901   } else if (opr1->is_address() && opr2->is_constant()) {
2902     LIR_Const* c = opr2->as_constant_ptr();
2903 #ifdef _LP64
2904     if (c->type() == T_OBJECT || c->type() == T_ARRAY || c->type() == T_VALUETYPE) {
2905       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "need to reverse");
2906       __ movoop(rscratch1, c->as_jobject());
2907     }
2908 #endif // LP64
2909     if (op->info() != NULL) {
2910       add_debug_info_for_null_check_here(op->info());
2911     }
2912     // special case: address - constant
2913     LIR_Address* addr = opr1->as_address_ptr();
2914     if (c->type() == T_INT) {
2915       __ cmpl(as_Address(addr), c->as_jint());
2916     } else if (c->type() == T_OBJECT || c->type() == T_ARRAY || c->type() == T_VALUETYPE) {
2917 #ifdef _LP64
2918       // %%% Make this explode if addr isn't reachable until we figure out a
2919       // better strategy by giving noreg as the temp for as_Address
2920       __ cmpoop(rscratch1, as_Address(addr, noreg));
2921 #else
2922       __ cmpoop(as_Address(addr), c->as_jobject());
2923 #endif // _LP64
2924     } else {
2925       ShouldNotReachHere();
2926     }
2927 
2928   } else {
2929     ShouldNotReachHere();
2930   }
2931 }
2932 
2933 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) {
2934   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2935     if (left->is_single_xmm()) {
2936       assert(right->is_single_xmm(), "must match");
2937       __ cmpss2int(left->as_xmm_float_reg(), right->as_xmm_float_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2938     } else if (left->is_double_xmm()) {
2939       assert(right->is_double_xmm(), "must match");
2940       __ cmpsd2int(left->as_xmm_double_reg(), right->as_xmm_double_reg(), dst->as_register(), code == lir_ucmp_fd2i);
2941 
2942     } else {
2943       assert(left->is_single_fpu() || left->is_double_fpu(), "must be");
2944       assert(right->is_single_fpu() || right->is_double_fpu(), "must match");
2945 
2946       assert(left->fpu() == 0, "left must be on TOS");
2947       __ fcmp2int(dst->as_register(), code == lir_ucmp_fd2i, right->fpu(),
2948                   op->fpu_pop_count() > 0, op->fpu_pop_count() > 1);
2949     }
2950   } else {
2951     assert(code == lir_cmp_l2i, "check");
2952 #ifdef _LP64
2953     Label done;
2954     Register dest = dst->as_register();
2955     __ cmpptr(left->as_register_lo(), right->as_register_lo());
2956     __ movl(dest, -1);
2957     __ jccb(Assembler::less, done);
2958     __ set_byte_if_not_zero(dest);
2959     __ movzbl(dest, dest);
2960     __ bind(done);
2961 #else
2962     __ lcmp2int(left->as_register_hi(),
2963                 left->as_register_lo(),
2964                 right->as_register_hi(),
2965                 right->as_register_lo());
2966     move_regs(left->as_register_hi(), dst->as_register());
2967 #endif // _LP64
2968   }
2969 }
2970 
2971 
2972 void LIR_Assembler::align_call(LIR_Code code) {
2973   // make sure that the displacement word of the call ends up word aligned
2974   int offset = __ offset();
2975   switch (code) {
2976   case lir_static_call:
2977   case lir_optvirtual_call:
2978   case lir_dynamic_call:
2979     offset += NativeCall::displacement_offset;
2980     break;
2981   case lir_icvirtual_call:
2982     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size;
2983     break;
2984   case lir_virtual_call:  // currently, sparc-specific for niagara
2985   default: ShouldNotReachHere();
2986   }
2987   __ align(BytesPerWord, offset);
2988 }
2989 
2990 
2991 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2992   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2993          "must be aligned");
2994   __ call(AddressLiteral(op->addr(), rtype));
2995   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2996 }
2997 
2998 
2999 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
3000   __ ic_call(op->addr());
3001   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
3002   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
3003          "must be aligned");
3004 }
3005 
3006 
3007 /* Currently, vtable-dispatch is only enabled for sparc platforms */
3008 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
3009   ShouldNotReachHere();
3010 }
3011 
3012 
3013 void LIR_Assembler::emit_static_call_stub() {
3014   address call_pc = __ pc();
3015   address stub = __ start_a_stub(call_stub_size());
3016   if (stub == NULL) {
3017     bailout("static call stub overflow");
3018     return;
3019   }
3020 
3021   int start = __ offset();
3022 
3023   // make sure that the displacement word of the call ends up word aligned
3024   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size + NativeCall::displacement_offset);
3025   __ relocate(static_stub_Relocation::spec(call_pc, false /* is_aot */));
3026   __ mov_metadata(rbx, (Metadata*)NULL);
3027   // must be set to -1 at code generation time
3028   assert(((__ offset() + 1) % BytesPerWord) == 0, "must be aligned");
3029   // On 64bit this will die since it will take a movq & jmp, must be only a jmp
3030   __ jump(RuntimeAddress(__ pc()));
3031 
3032   if (UseAOT) {
3033     // Trampoline to aot code
3034     __ relocate(static_stub_Relocation::spec(call_pc, true /* is_aot */));
3035 #ifdef _LP64
3036     __ mov64(rax, CONST64(0));  // address is zapped till fixup time.
3037 #else
3038     __ movl(rax, 0xdeadffff);  // address is zapped till fixup time.
3039 #endif
3040     __ jmp(rax);
3041   }
3042   assert(__ offset() - start <= call_stub_size(), "stub too big");
3043   __ end_a_stub();
3044 }
3045 
3046 
3047 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
3048   assert(exceptionOop->as_register() == rax, "must match");
3049   assert(exceptionPC->as_register() == rdx, "must match");
3050 
3051   // exception object is not added to oop map by LinearScan
3052   // (LinearScan assumes that no oops are in fixed registers)
3053   info->add_register_oop(exceptionOop);
3054   Runtime1::StubID unwind_id;
3055 
3056   // get current pc information
3057   // pc is only needed if the method has an exception handler, the unwind code does not need it.
3058   int pc_for_athrow_offset = __ offset();
3059   InternalAddress pc_for_athrow(__ pc());
3060   __ lea(exceptionPC->as_register(), pc_for_athrow);
3061   add_call_info(pc_for_athrow_offset, info); // for exception handler
3062 
3063   __ verify_not_null_oop(rax);
3064   // search an exception handler (rax: exception oop, rdx: throwing pc)
3065   if (compilation()->has_fpu_code()) {
3066     unwind_id = Runtime1::handle_exception_id;
3067   } else {
3068     unwind_id = Runtime1::handle_exception_nofpu_id;
3069   }
3070   __ call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
3071 
3072   // enough room for two byte trap
3073   __ nop();
3074 }
3075 
3076 
3077 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
3078   assert(exceptionOop->as_register() == rax, "must match");
3079 
3080   __ jmp(_unwind_handler_entry);
3081 }
3082 
3083 
3084 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
3085 
3086   // optimized version for linear scan:
3087   // * count must be already in ECX (guaranteed by LinearScan)
3088   // * left and dest must be equal
3089   // * tmp must be unused
3090   assert(count->as_register() == SHIFT_count, "count must be in ECX");
3091   assert(left == dest, "left and dest must be equal");
3092   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
3093 
3094   if (left->is_single_cpu()) {
3095     Register value = left->as_register();
3096     assert(value != SHIFT_count, "left cannot be ECX");
3097 
3098     switch (code) {
3099       case lir_shl:  __ shll(value); break;
3100       case lir_shr:  __ sarl(value); break;
3101       case lir_ushr: __ shrl(value); break;
3102       default: ShouldNotReachHere();
3103     }
3104   } else if (left->is_double_cpu()) {
3105     Register lo = left->as_register_lo();
3106     Register hi = left->as_register_hi();
3107     assert(lo != SHIFT_count && hi != SHIFT_count, "left cannot be ECX");
3108 #ifdef _LP64
3109     switch (code) {
3110       case lir_shl:  __ shlptr(lo);        break;
3111       case lir_shr:  __ sarptr(lo);        break;
3112       case lir_ushr: __ shrptr(lo);        break;
3113       default: ShouldNotReachHere();
3114     }
3115 #else
3116 
3117     switch (code) {
3118       case lir_shl:  __ lshl(hi, lo);        break;
3119       case lir_shr:  __ lshr(hi, lo, true);  break;
3120       case lir_ushr: __ lshr(hi, lo, false); break;
3121       default: ShouldNotReachHere();
3122     }
3123 #endif // LP64
3124   } else {
3125     ShouldNotReachHere();
3126   }
3127 }
3128 
3129 
3130 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
3131   if (dest->is_single_cpu()) {
3132     // first move left into dest so that left is not destroyed by the shift
3133     Register value = dest->as_register();
3134     count = count & 0x1F; // Java spec
3135 
3136     move_regs(left->as_register(), value);
3137     switch (code) {
3138       case lir_shl:  __ shll(value, count); break;
3139       case lir_shr:  __ sarl(value, count); break;
3140       case lir_ushr: __ shrl(value, count); break;
3141       default: ShouldNotReachHere();
3142     }
3143   } else if (dest->is_double_cpu()) {
3144 #ifndef _LP64
3145     Unimplemented();
3146 #else
3147     // first move left into dest so that left is not destroyed by the shift
3148     Register value = dest->as_register_lo();
3149     count = count & 0x1F; // Java spec
3150 
3151     move_regs(left->as_register_lo(), value);
3152     switch (code) {
3153       case lir_shl:  __ shlptr(value, count); break;
3154       case lir_shr:  __ sarptr(value, count); break;
3155       case lir_ushr: __ shrptr(value, count); break;
3156       default: ShouldNotReachHere();
3157     }
3158 #endif // _LP64
3159   } else {
3160     ShouldNotReachHere();
3161   }
3162 }
3163 
3164 
3165 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
3166   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3167   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3168   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3169   __ movptr (Address(rsp, offset_from_rsp_in_bytes), r);
3170 }
3171 
3172 
3173 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
3174   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3175   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3176   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3177   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
3178 }
3179 
3180 
3181 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
3182   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3183   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3184   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3185   __ movoop (Address(rsp, offset_from_rsp_in_bytes), o);
3186 }
3187 
3188 
3189 void LIR_Assembler::store_parameter(Metadata* m,  int offset_from_rsp_in_words) {
3190   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
3191   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
3192   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
3193   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m);
3194 }
3195 
3196 
3197 void LIR_Assembler::arraycopy_valuetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
3198   if (null_check) {
3199     __ testptr(obj, obj);
3200     __ jcc(Assembler::zero, *slow_path->entry());
3201   }
3202   __ load_storage_props(tmp, obj);
3203   if (is_dest) {
3204     // We also take slow path if it's a null_free destination array, just in case the source array
3205     // contains NULLs.
3206     __ testb(tmp, ArrayStorageProperties::flattened_value | ArrayStorageProperties::null_free_value);
3207   } else {
3208     __ testb(tmp, ArrayStorageProperties::flattened_value);
3209   }
3210   __ jcc(Assembler::notEqual, *slow_path->entry());
3211 }
3212 
3213 
3214 // This code replaces a call to arraycopy; no exception may
3215 // be thrown in this code, they must be thrown in the System.arraycopy
3216 // activation frame; we could save some checks if this would not be the case
3217 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
3218   ciArrayKlass* default_type = op->expected_type();
3219   Register src = op->src()->as_register();
3220   Register dst = op->dst()->as_register();
3221   Register src_pos = op->src_pos()->as_register();
3222   Register dst_pos = op->dst_pos()->as_register();
3223   Register length  = op->length()->as_register();
3224   Register tmp = op->tmp()->as_register();
3225 
3226   __ resolve(ACCESS_READ, src);
3227   __ resolve(ACCESS_WRITE, dst);
3228 
3229   CodeStub* stub = op->stub();
3230   int flags = op->flags();
3231   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
3232   if (basic_type == T_ARRAY || basic_type == T_VALUETYPE) basic_type = T_OBJECT;
3233 
3234   if (flags & LIR_OpArrayCopy::always_slow_path) {
3235     __ jmp(*stub->entry());
3236     __ bind(*stub->continuation());
3237     return;
3238   }
3239 
3240   if (flags & LIR_OpArrayCopy::src_valuetype_check) {
3241     arraycopy_valuetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
3242   }
3243 
3244   if (flags & LIR_OpArrayCopy::dst_valuetype_check) {
3245     arraycopy_valuetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
3246   }
3247 
3248   // if we don't know anything, just go through the generic arraycopy
3249   if (default_type == NULL) {
3250     // save outgoing arguments on stack in case call to System.arraycopy is needed
3251     // HACK ALERT. This code used to push the parameters in a hardwired fashion
3252     // for interpreter calling conventions. Now we have to do it in new style conventions.
3253     // For the moment until C1 gets the new register allocator I just force all the
3254     // args to the right place (except the register args) and then on the back side
3255     // reload the register args properly if we go slow path. Yuck
3256 
3257     // These are proper for the calling convention
3258     store_parameter(length, 2);
3259     store_parameter(dst_pos, 1);
3260     store_parameter(dst, 0);
3261 
3262     // these are just temporary placements until we need to reload
3263     store_parameter(src_pos, 3);
3264     store_parameter(src, 4);
3265     NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
3266 
3267     address copyfunc_addr = StubRoutines::generic_arraycopy();
3268     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
3269 
3270     // pass arguments: may push as this is not a safepoint; SP must be fix at each safepoint
3271 #ifdef _LP64
3272     // The arguments are in java calling convention so we can trivially shift them to C
3273     // convention
3274     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
3275     __ mov(c_rarg0, j_rarg0);
3276     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
3277     __ mov(c_rarg1, j_rarg1);
3278     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
3279     __ mov(c_rarg2, j_rarg2);
3280     assert_different_registers(c_rarg3, j_rarg4);
3281     __ mov(c_rarg3, j_rarg3);
3282 #ifdef _WIN64
3283     // Allocate abi space for args but be sure to keep stack aligned
3284     __ subptr(rsp, 6*wordSize);
3285     store_parameter(j_rarg4, 4);
3286 #ifndef PRODUCT
3287     if (PrintC1Statistics) {
3288       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3289     }
3290 #endif
3291     __ call(RuntimeAddress(copyfunc_addr));
3292     __ addptr(rsp, 6*wordSize);
3293 #else
3294     __ mov(c_rarg4, j_rarg4);
3295 #ifndef PRODUCT
3296     if (PrintC1Statistics) {
3297       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3298     }
3299 #endif
3300     __ call(RuntimeAddress(copyfunc_addr));
3301 #endif // _WIN64
3302 #else
3303     __ push(length);
3304     __ push(dst_pos);
3305     __ push(dst);
3306     __ push(src_pos);
3307     __ push(src);
3308 
3309 #ifndef PRODUCT
3310     if (PrintC1Statistics) {
3311       __ incrementl(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
3312     }
3313 #endif
3314     __ call_VM_leaf(copyfunc_addr, 5); // removes pushed parameter from the stack
3315 
3316 #endif // _LP64
3317 
3318     __ cmpl(rax, 0);
3319     __ jcc(Assembler::equal, *stub->continuation());
3320 
3321     __ mov(tmp, rax);
3322     __ xorl(tmp, -1);
3323 
3324     // Reload values from the stack so they are where the stub
3325     // expects them.
3326     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3327     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3328     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3329     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3330     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3331 
3332     __ subl(length, tmp);
3333     __ addl(src_pos, tmp);
3334     __ addl(dst_pos, tmp);
3335     __ jmp(*stub->entry());
3336 
3337     __ bind(*stub->continuation());
3338     return;
3339   }
3340 
3341   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
3342 
3343   int elem_size = type2aelembytes(basic_type);
3344   Address::ScaleFactor scale;
3345 
3346   switch (elem_size) {
3347     case 1 :
3348       scale = Address::times_1;
3349       break;
3350     case 2 :
3351       scale = Address::times_2;
3352       break;
3353     case 4 :
3354       scale = Address::times_4;
3355       break;
3356     case 8 :
3357       scale = Address::times_8;
3358       break;
3359     default:
3360       scale = Address::no_scale;
3361       ShouldNotReachHere();
3362   }
3363 
3364   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
3365   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
3366   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
3367   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
3368 
3369   // length and pos's are all sign extended at this point on 64bit
3370 
3371   // test for NULL
3372   if (flags & LIR_OpArrayCopy::src_null_check) {
3373     __ testptr(src, src);
3374     __ jcc(Assembler::zero, *stub->entry());
3375   }
3376   if (flags & LIR_OpArrayCopy::dst_null_check) {
3377     __ testptr(dst, dst);
3378     __ jcc(Assembler::zero, *stub->entry());
3379   }
3380 
3381   // If the compiler was not able to prove that exact type of the source or the destination
3382   // of the arraycopy is an array type, check at runtime if the source or the destination is
3383   // an instance type.
3384   if (flags & LIR_OpArrayCopy::type_check) {
3385     if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3386       __ load_klass(tmp, dst);
3387       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3388       __ jcc(Assembler::greaterEqual, *stub->entry());
3389     }
3390 
3391     if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3392       __ load_klass(tmp, src);
3393       __ cmpl(Address(tmp, in_bytes(Klass::layout_helper_offset())), Klass::_lh_neutral_value);
3394       __ jcc(Assembler::greaterEqual, *stub->entry());
3395     }
3396   }
3397 
3398   // check if negative
3399   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
3400     __ testl(src_pos, src_pos);
3401     __ jcc(Assembler::less, *stub->entry());
3402   }
3403   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
3404     __ testl(dst_pos, dst_pos);
3405     __ jcc(Assembler::less, *stub->entry());
3406   }
3407 
3408   if (flags & LIR_OpArrayCopy::src_range_check) {
3409     __ lea(tmp, Address(src_pos, length, Address::times_1, 0));
3410     __ cmpl(tmp, src_length_addr);
3411     __ jcc(Assembler::above, *stub->entry());
3412   }
3413   if (flags & LIR_OpArrayCopy::dst_range_check) {
3414     __ lea(tmp, Address(dst_pos, length, Address::times_1, 0));
3415     __ cmpl(tmp, dst_length_addr);
3416     __ jcc(Assembler::above, *stub->entry());
3417   }
3418 
3419   if (flags & LIR_OpArrayCopy::length_positive_check) {
3420     __ testl(length, length);
3421     __ jcc(Assembler::less, *stub->entry());
3422   }
3423 
3424 #ifdef _LP64
3425   __ movl2ptr(src_pos, src_pos); //higher 32bits must be null
3426   __ movl2ptr(dst_pos, dst_pos); //higher 32bits must be null
3427 #endif
3428 
3429   if (flags & LIR_OpArrayCopy::type_check) {
3430     // We don't know the array types are compatible
3431     if (basic_type != T_OBJECT) {
3432       // Simple test for basic type arrays
3433       if (UseCompressedClassPointers) {
3434         __ movl(tmp, src_klass_addr);
3435         __ cmpl(tmp, dst_klass_addr);
3436       } else {
3437         __ movptr(tmp, src_klass_addr);
3438         __ cmpptr(tmp, dst_klass_addr);
3439       }
3440       __ jcc(Assembler::notEqual, *stub->entry());
3441     } else {
3442       // For object arrays, if src is a sub class of dst then we can
3443       // safely do the copy.
3444       Label cont, slow;
3445 
3446       __ push(src);
3447       __ push(dst);
3448 
3449       __ load_klass(src, src);
3450       __ load_klass(dst, dst);
3451 
3452       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
3453 
3454       __ push(src);
3455       __ push(dst);
3456       __ call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
3457       __ pop(dst);
3458       __ pop(src);
3459 
3460       __ cmpl(src, 0);
3461       __ jcc(Assembler::notEqual, cont);
3462 
3463       __ bind(slow);
3464       __ pop(dst);
3465       __ pop(src);
3466 
3467       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
3468       if (copyfunc_addr != NULL) { // use stub if available
3469         // src is not a sub class of dst so we have to do a
3470         // per-element check.
3471 
3472         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
3473         if ((flags & mask) != mask) {
3474           // Check that at least both of them object arrays.
3475           assert(flags & mask, "one of the two should be known to be an object array");
3476 
3477           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
3478             __ load_klass(tmp, src);
3479           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
3480             __ load_klass(tmp, dst);
3481           }
3482           int lh_offset = in_bytes(Klass::layout_helper_offset());
3483           Address klass_lh_addr(tmp, lh_offset);
3484           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
3485           __ cmpl(klass_lh_addr, objArray_lh);
3486           __ jcc(Assembler::notEqual, *stub->entry());
3487         }
3488 
3489        // Spill because stubs can use any register they like and it's
3490        // easier to restore just those that we care about.
3491        store_parameter(dst, 0);
3492        store_parameter(dst_pos, 1);
3493        store_parameter(length, 2);
3494        store_parameter(src_pos, 3);
3495        store_parameter(src, 4);
3496 
3497 #ifndef _LP64
3498         __ movptr(tmp, dst_klass_addr);
3499         __ movptr(tmp, Address(tmp, ObjArrayKlass::element_klass_offset()));
3500         __ push(tmp);
3501         __ movl(tmp, Address(tmp, Klass::super_check_offset_offset()));
3502         __ push(tmp);
3503         __ push(length);
3504         __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3505         __ push(tmp);
3506         __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3507         __ push(tmp);
3508 
3509         __ call_VM_leaf(copyfunc_addr, 5);
3510 #else
3511         __ movl2ptr(length, length); //higher 32bits must be null
3512 
3513         __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3514         assert_different_registers(c_rarg0, dst, dst_pos, length);
3515         __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3516         assert_different_registers(c_rarg1, dst, length);
3517 
3518         __ mov(c_rarg2, length);
3519         assert_different_registers(c_rarg2, dst);
3520 
3521 #ifdef _WIN64
3522         // Allocate abi space for args but be sure to keep stack aligned
3523         __ subptr(rsp, 6*wordSize);
3524         __ load_klass(c_rarg3, dst);
3525         __ movptr(c_rarg3, Address(c_rarg3, ObjArrayKlass::element_klass_offset()));
3526         store_parameter(c_rarg3, 4);
3527         __ movl(c_rarg3, Address(c_rarg3, Klass::super_check_offset_offset()));
3528         __ call(RuntimeAddress(copyfunc_addr));
3529         __ addptr(rsp, 6*wordSize);
3530 #else
3531         __ load_klass(c_rarg4, dst);
3532         __ movptr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
3533         __ movl(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
3534         __ call(RuntimeAddress(copyfunc_addr));
3535 #endif
3536 
3537 #endif
3538 
3539 #ifndef PRODUCT
3540         if (PrintC1Statistics) {
3541           Label failed;
3542           __ testl(rax, rax);
3543           __ jcc(Assembler::notZero, failed);
3544           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
3545           __ bind(failed);
3546         }
3547 #endif
3548 
3549         __ testl(rax, rax);
3550         __ jcc(Assembler::zero, *stub->continuation());
3551 
3552 #ifndef PRODUCT
3553         if (PrintC1Statistics) {
3554           __ incrementl(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
3555         }
3556 #endif
3557 
3558         __ mov(tmp, rax);
3559 
3560         __ xorl(tmp, -1);
3561 
3562         // Restore previously spilled arguments
3563         __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
3564         __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
3565         __ movptr   (length,  Address(rsp, 2*BytesPerWord));
3566         __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
3567         __ movptr   (src,     Address(rsp, 4*BytesPerWord));
3568 
3569 
3570         __ subl(length, tmp);
3571         __ addl(src_pos, tmp);
3572         __ addl(dst_pos, tmp);
3573       }
3574 
3575       __ jmp(*stub->entry());
3576 
3577       __ bind(cont);
3578       __ pop(dst);
3579       __ pop(src);
3580     }
3581   }
3582 
3583 #ifdef ASSERT
3584   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
3585     // Sanity check the known type with the incoming class.  For the
3586     // primitive case the types must match exactly with src.klass and
3587     // dst.klass each exactly matching the default type.  For the
3588     // object array case, if no type check is needed then either the
3589     // dst type is exactly the expected type and the src type is a
3590     // subtype which we can't check or src is the same array as dst
3591     // but not necessarily exactly of type default_type.
3592     Label known_ok, halt;
3593     __ mov_metadata(tmp, default_type->constant_encoding());
3594 #ifdef _LP64
3595     if (UseCompressedClassPointers) {
3596       __ encode_klass_not_null(tmp);
3597     }
3598 #endif
3599 
3600     if (basic_type != T_OBJECT) {
3601 
3602       if (UseCompressedClassPointers)          __ cmpl(tmp, dst_klass_addr);
3603       else                   __ cmpptr(tmp, dst_klass_addr);
3604       __ jcc(Assembler::notEqual, halt);
3605       if (UseCompressedClassPointers)          __ cmpl(tmp, src_klass_addr);
3606       else                   __ cmpptr(tmp, src_klass_addr);
3607       __ jcc(Assembler::equal, known_ok);
3608     } else {
3609       if (UseCompressedClassPointers)          __ cmpl(tmp, dst_klass_addr);
3610       else                   __ cmpptr(tmp, dst_klass_addr);
3611       __ jcc(Assembler::equal, known_ok);
3612       __ cmpptr(src, dst);
3613       __ jcc(Assembler::equal, known_ok);
3614     }
3615     __ bind(halt);
3616     __ stop("incorrect type information in arraycopy");
3617     __ bind(known_ok);
3618   }
3619 #endif
3620 
3621 #ifndef PRODUCT
3622   if (PrintC1Statistics) {
3623     __ incrementl(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
3624   }
3625 #endif
3626 
3627 #ifdef _LP64
3628   assert_different_registers(c_rarg0, dst, dst_pos, length);
3629   __ lea(c_rarg0, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3630   assert_different_registers(c_rarg1, length);
3631   __ lea(c_rarg1, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3632   __ mov(c_rarg2, length);
3633 
3634 #else
3635   __ lea(tmp, Address(src, src_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3636   store_parameter(tmp, 0);
3637   __ lea(tmp, Address(dst, dst_pos, scale, arrayOopDesc::base_offset_in_bytes(basic_type)));
3638   store_parameter(tmp, 1);
3639   store_parameter(length, 2);
3640 #endif // _LP64
3641 
3642   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
3643   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
3644   const char *name;
3645   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
3646   __ call_VM_leaf(entry, 0);
3647 
3648   __ bind(*stub->continuation());
3649 }
3650 
3651 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
3652   assert(op->crc()->is_single_cpu(),  "crc must be register");
3653   assert(op->val()->is_single_cpu(),  "byte value must be register");
3654   assert(op->result_opr()->is_single_cpu(), "result must be register");
3655   Register crc = op->crc()->as_register();
3656   Register val = op->val()->as_register();
3657   Register res = op->result_opr()->as_register();
3658 
3659   assert_different_registers(val, crc, res);
3660 
3661   __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
3662   __ notl(crc); // ~crc
3663   __ update_byte_crc32(crc, val, res);
3664   __ notl(crc); // ~crc
3665   __ mov(res, crc);
3666 }
3667 
3668 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
3669   Register obj = op->obj_opr()->as_register();  // may not be an oop
3670   Register hdr = op->hdr_opr()->as_register();
3671   Register lock = op->lock_opr()->as_register();
3672   if (!UseFastLocking) {
3673     __ jmp(*op->stub()->entry());
3674   } else if (op->code() == lir_lock) {
3675     Register scratch = noreg;
3676     if (UseBiasedLocking) {
3677       scratch = op->scratch_opr()->as_register();
3678     }
3679     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3680     __ resolve(ACCESS_READ | ACCESS_WRITE, obj);
3681     // add debug info for NullPointerException only if one is possible
3682     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
3683     if (op->info() != NULL) {
3684       add_debug_info_for_null_check(null_check_offset, op->info());
3685     }
3686     // done
3687   } else if (op->code() == lir_unlock) {
3688     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3689     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3690   } else {
3691     Unimplemented();
3692   }
3693   __ bind(*op->stub()->continuation());
3694 }
3695 
3696 
3697 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3698   ciMethod* method = op->profiled_method();
3699   int bci          = op->profiled_bci();
3700   ciMethod* callee = op->profiled_callee();
3701 
3702   // Update counter for all call types
3703   ciMethodData* md = method->method_data_or_null();
3704   assert(md != NULL, "Sanity");
3705   ciProfileData* data = md->bci_to_data(bci);
3706   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
3707   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
3708   Register mdo  = op->mdo()->as_register();
3709   __ mov_metadata(mdo, md->constant_encoding());
3710   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
3711   // Perform additional virtual call profiling for invokevirtual and
3712   // invokeinterface bytecodes
3713   if (op->should_profile_receiver_type()) {
3714     assert(op->recv()->is_single_cpu(), "recv must be allocated");
3715     Register recv = op->recv()->as_register();
3716     assert_different_registers(mdo, recv);
3717     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3718     ciKlass* known_klass = op->known_holder();
3719     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
3720       // We know the type that will be seen at this call site; we can
3721       // statically update the MethodData* rather than needing to do
3722       // dynamic tests on the receiver type
3723 
3724       // NOTE: we should probably put a lock around this search to
3725       // avoid collisions by concurrent compilations
3726       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3727       uint i;
3728       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3729         ciKlass* receiver = vc_data->receiver(i);
3730         if (known_klass->equals(receiver)) {
3731           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3732           __ addptr(data_addr, DataLayout::counter_increment);
3733           return;
3734         }
3735       }
3736 
3737       // Receiver type not found in profile data; select an empty slot
3738 
3739       // Note that this is less efficient than it should be because it
3740       // always does a write to the receiver part of the
3741       // VirtualCallData rather than just the first time
3742       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3743         ciKlass* receiver = vc_data->receiver(i);
3744         if (receiver == NULL) {
3745           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
3746           __ mov_metadata(recv_addr, known_klass->constant_encoding());
3747           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
3748           __ addptr(data_addr, DataLayout::counter_increment);
3749           return;
3750         }
3751       }
3752     } else {
3753       __ load_klass(recv, recv);
3754       Label update_done;
3755       type_profile_helper(mdo, md, data, recv, &update_done);
3756       // Receiver did not match any saved receiver and there is no empty row for it.
3757       // Increment total counter to indicate polymorphic case.
3758       __ addptr(counter_addr, DataLayout::counter_increment);
3759 
3760       __ bind(update_done);
3761     }
3762   } else {
3763     // Static call
3764     __ addptr(counter_addr, DataLayout::counter_increment);
3765   }
3766 }
3767 
3768 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3769   Register obj = op->obj()->as_register();
3770   Register tmp = op->tmp()->as_pointer_register();
3771   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3772   ciKlass* exact_klass = op->exact_klass();
3773   intptr_t current_klass = op->current_klass();
3774   bool not_null = op->not_null();
3775   bool no_conflict = op->no_conflict();
3776 
3777   Label update, next, none;
3778 
3779   bool do_null = !not_null;
3780   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3781   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3782 
3783   assert(do_null || do_update, "why are we here?");
3784   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3785 
3786   __ verify_oop(obj);
3787 
3788   if (tmp != obj) {
3789     __ mov(tmp, obj);
3790   }
3791   if (do_null) {
3792     __ testptr(tmp, tmp);
3793     __ jccb(Assembler::notZero, update);
3794     if (!TypeEntries::was_null_seen(current_klass)) {
3795       __ orptr(mdo_addr, TypeEntries::null_seen);
3796     }
3797     if (do_update) {
3798 #ifndef ASSERT
3799       __ jmpb(next);
3800     }
3801 #else
3802       __ jmp(next);
3803     }
3804   } else {
3805     __ testptr(tmp, tmp);
3806     __ jcc(Assembler::notZero, update);
3807     __ stop("unexpect null obj");
3808 #endif
3809   }
3810 
3811   __ bind(update);
3812 
3813   if (do_update) {
3814 #ifdef ASSERT
3815     if (exact_klass != NULL) {
3816       Label ok;
3817       __ load_klass(tmp, tmp);
3818       __ push(tmp);
3819       __ mov_metadata(tmp, exact_klass->constant_encoding());
3820       __ cmpptr(tmp, Address(rsp, 0));
3821       __ jcc(Assembler::equal, ok);
3822       __ stop("exact klass and actual klass differ");
3823       __ bind(ok);
3824       __ pop(tmp);
3825     }
3826 #endif
3827     if (!no_conflict) {
3828       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
3829         if (exact_klass != NULL) {
3830           __ mov_metadata(tmp, exact_klass->constant_encoding());
3831         } else {
3832           __ load_klass(tmp, tmp);
3833         }
3834 
3835         __ xorptr(tmp, mdo_addr);
3836         __ testptr(tmp, TypeEntries::type_klass_mask);
3837         // klass seen before, nothing to do. The unknown bit may have been
3838         // set already but no need to check.
3839         __ jccb(Assembler::zero, next);
3840 
3841         __ testptr(tmp, TypeEntries::type_unknown);
3842         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3843 
3844         if (TypeEntries::is_type_none(current_klass)) {
3845           __ cmpptr(mdo_addr, 0);
3846           __ jccb(Assembler::equal, none);
3847           __ cmpptr(mdo_addr, TypeEntries::null_seen);
3848           __ jccb(Assembler::equal, none);
3849           // There is a chance that the checks above (re-reading profiling
3850           // data from memory) fail if another thread has just set the
3851           // profiling to this obj's klass
3852           __ xorptr(tmp, mdo_addr);
3853           __ testptr(tmp, TypeEntries::type_klass_mask);
3854           __ jccb(Assembler::zero, next);
3855         }
3856       } else {
3857         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3858                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3859 
3860         __ movptr(tmp, mdo_addr);
3861         __ testptr(tmp, TypeEntries::type_unknown);
3862         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3863       }
3864 
3865       // different than before. Cannot keep accurate profile.
3866       __ orptr(mdo_addr, TypeEntries::type_unknown);
3867 
3868       if (TypeEntries::is_type_none(current_klass)) {
3869         __ jmpb(next);
3870 
3871         __ bind(none);
3872         // first time here. Set profile type.
3873         __ movptr(mdo_addr, tmp);
3874       }
3875     } else {
3876       // There's a single possible klass at this profile point
3877       assert(exact_klass != NULL, "should be");
3878       if (TypeEntries::is_type_none(current_klass)) {
3879         __ mov_metadata(tmp, exact_klass->constant_encoding());
3880         __ xorptr(tmp, mdo_addr);
3881         __ testptr(tmp, TypeEntries::type_klass_mask);
3882 #ifdef ASSERT
3883         __ jcc(Assembler::zero, next);
3884 
3885         {
3886           Label ok;
3887           __ push(tmp);
3888           __ cmpptr(mdo_addr, 0);
3889           __ jcc(Assembler::equal, ok);
3890           __ cmpptr(mdo_addr, TypeEntries::null_seen);
3891           __ jcc(Assembler::equal, ok);
3892           // may have been set by another thread
3893           __ mov_metadata(tmp, exact_klass->constant_encoding());
3894           __ xorptr(tmp, mdo_addr);
3895           __ testptr(tmp, TypeEntries::type_mask);
3896           __ jcc(Assembler::zero, ok);
3897 
3898           __ stop("unexpected profiling mismatch");
3899           __ bind(ok);
3900           __ pop(tmp);
3901         }
3902 #else
3903         __ jccb(Assembler::zero, next);
3904 #endif
3905         // first time here. Set profile type.
3906         __ movptr(mdo_addr, tmp);
3907       } else {
3908         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3909                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3910 
3911         __ movptr(tmp, mdo_addr);
3912         __ testptr(tmp, TypeEntries::type_unknown);
3913         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3914 
3915         __ orptr(mdo_addr, TypeEntries::type_unknown);
3916       }
3917     }
3918 
3919     __ bind(next);
3920   }
3921 }
3922 
3923 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3924   Unimplemented();
3925 }
3926 
3927 
3928 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3929   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3930 }
3931 
3932 
3933 void LIR_Assembler::align_backward_branch_target() {
3934   __ align(BytesPerWord);
3935 }
3936 
3937 
3938 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3939   if (left->is_single_cpu()) {
3940     __ negl(left->as_register());
3941     move_regs(left->as_register(), dest->as_register());
3942 
3943   } else if (left->is_double_cpu()) {
3944     Register lo = left->as_register_lo();
3945 #ifdef _LP64
3946     Register dst = dest->as_register_lo();
3947     __ movptr(dst, lo);
3948     __ negptr(dst);
3949 #else
3950     Register hi = left->as_register_hi();
3951     __ lneg(hi, lo);
3952     if (dest->as_register_lo() == hi) {
3953       assert(dest->as_register_hi() != lo, "destroying register");
3954       move_regs(hi, dest->as_register_hi());
3955       move_regs(lo, dest->as_register_lo());
3956     } else {
3957       move_regs(lo, dest->as_register_lo());
3958       move_regs(hi, dest->as_register_hi());
3959     }
3960 #endif // _LP64
3961 
3962   } else if (dest->is_single_xmm()) {
3963 #ifdef _LP64
3964     if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3965       assert(tmp->is_valid(), "need temporary");
3966       assert_different_registers(left->as_xmm_float_reg(), tmp->as_xmm_float_reg());
3967       __ vpxor(dest->as_xmm_float_reg(), tmp->as_xmm_float_reg(), left->as_xmm_float_reg(), 2);
3968     }
3969     else
3970 #endif
3971     {
3972       assert(!tmp->is_valid(), "do not need temporary");
3973       if (left->as_xmm_float_reg() != dest->as_xmm_float_reg()) {
3974         __ movflt(dest->as_xmm_float_reg(), left->as_xmm_float_reg());
3975       }
3976       __ xorps(dest->as_xmm_float_reg(),
3977                ExternalAddress((address)float_signflip_pool));
3978     }
3979   } else if (dest->is_double_xmm()) {
3980 #ifdef _LP64
3981     if (UseAVX > 2 && !VM_Version::supports_avx512vl()) {
3982       assert(tmp->is_valid(), "need temporary");
3983       assert_different_registers(left->as_xmm_double_reg(), tmp->as_xmm_double_reg());
3984       __ vpxor(dest->as_xmm_double_reg(), tmp->as_xmm_double_reg(), left->as_xmm_double_reg(), 2);
3985     }
3986     else
3987 #endif
3988     {
3989       assert(!tmp->is_valid(), "do not need temporary");
3990       if (left->as_xmm_double_reg() != dest->as_xmm_double_reg()) {
3991         __ movdbl(dest->as_xmm_double_reg(), left->as_xmm_double_reg());
3992       }
3993       __ xorpd(dest->as_xmm_double_reg(),
3994                ExternalAddress((address)double_signflip_pool));
3995     }
3996   } else if (left->is_single_fpu() || left->is_double_fpu()) {
3997     assert(left->fpu() == 0, "arg must be on TOS");
3998     assert(dest->fpu() == 0, "dest must be TOS");
3999     __ fchs();
4000 
4001   } else {
4002     ShouldNotReachHere();
4003   }
4004 }
4005 
4006 
4007 void LIR_Assembler::leal(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
4008   assert(src->is_address(), "must be an address");
4009   assert(dest->is_register(), "must be a register");
4010 
4011   PatchingStub* patch = NULL;
4012   if (patch_code != lir_patch_none) {
4013     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
4014   }
4015 
4016   Register reg = dest->as_pointer_register();
4017   LIR_Address* addr = src->as_address_ptr();
4018   __ lea(reg, as_Address(addr));
4019 
4020   if (patch != NULL) {
4021     patching_epilog(patch, patch_code, addr->base()->as_register(), info);
4022   }
4023 }
4024 
4025 
4026 
4027 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
4028   assert(!tmp->is_valid(), "don't need temporary");
4029   __ call(RuntimeAddress(dest));
4030   if (info != NULL) {
4031     add_call_info_here(info);
4032   }
4033 }
4034 
4035 
4036 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
4037   assert(type == T_LONG, "only for volatile long fields");
4038 
4039   if (info != NULL) {
4040     add_debug_info_for_null_check_here(info);
4041   }
4042 
4043   if (src->is_double_xmm()) {
4044     if (dest->is_double_cpu()) {
4045 #ifdef _LP64
4046       __ movdq(dest->as_register_lo(), src->as_xmm_double_reg());
4047 #else
4048       __ movdl(dest->as_register_lo(), src->as_xmm_double_reg());
4049       __ psrlq(src->as_xmm_double_reg(), 32);
4050       __ movdl(dest->as_register_hi(), src->as_xmm_double_reg());
4051 #endif // _LP64
4052     } else if (dest->is_double_stack()) {
4053       __ movdbl(frame_map()->address_for_slot(dest->double_stack_ix()), src->as_xmm_double_reg());
4054     } else if (dest->is_address()) {
4055       __ movdbl(as_Address(dest->as_address_ptr()), src->as_xmm_double_reg());
4056     } else {
4057       ShouldNotReachHere();
4058     }
4059 
4060   } else if (dest->is_double_xmm()) {
4061     if (src->is_double_stack()) {
4062       __ movdbl(dest->as_xmm_double_reg(), frame_map()->address_for_slot(src->double_stack_ix()));
4063     } else if (src->is_address()) {
4064       __ movdbl(dest->as_xmm_double_reg(), as_Address(src->as_address_ptr()));
4065     } else {
4066       ShouldNotReachHere();
4067     }
4068 
4069   } else if (src->is_double_fpu()) {
4070     assert(src->fpu_regnrLo() == 0, "must be TOS");
4071     if (dest->is_double_stack()) {
4072       __ fistp_d(frame_map()->address_for_slot(dest->double_stack_ix()));
4073     } else if (dest->is_address()) {
4074       __ fistp_d(as_Address(dest->as_address_ptr()));
4075     } else {
4076       ShouldNotReachHere();
4077     }
4078 
4079   } else if (dest->is_double_fpu()) {
4080     assert(dest->fpu_regnrLo() == 0, "must be TOS");
4081     if (src->is_double_stack()) {
4082       __ fild_d(frame_map()->address_for_slot(src->double_stack_ix()));
4083     } else if (src->is_address()) {
4084       __ fild_d(as_Address(src->as_address_ptr()));
4085     } else {
4086       ShouldNotReachHere();
4087     }
4088   } else {
4089     ShouldNotReachHere();
4090   }
4091 }
4092 
4093 #ifdef ASSERT
4094 // emit run-time assertion
4095 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
4096   assert(op->code() == lir_assert, "must be");
4097 
4098   if (op->in_opr1()->is_valid()) {
4099     assert(op->in_opr2()->is_valid(), "both operands must be valid");
4100     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
4101   } else {
4102     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
4103     assert(op->condition() == lir_cond_always, "no other conditions allowed");
4104   }
4105 
4106   Label ok;
4107   if (op->condition() != lir_cond_always) {
4108     Assembler::Condition acond = Assembler::zero;
4109     switch (op->condition()) {
4110       case lir_cond_equal:        acond = Assembler::equal;       break;
4111       case lir_cond_notEqual:     acond = Assembler::notEqual;    break;
4112       case lir_cond_less:         acond = Assembler::less;        break;
4113       case lir_cond_lessEqual:    acond = Assembler::lessEqual;   break;
4114       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;break;
4115       case lir_cond_greater:      acond = Assembler::greater;     break;
4116       case lir_cond_belowEqual:   acond = Assembler::belowEqual;  break;
4117       case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;  break;
4118       default:                    ShouldNotReachHere();
4119     }
4120     __ jcc(acond, ok);
4121   }
4122   if (op->halt()) {
4123     const char* str = __ code_string(op->msg());
4124     __ stop(str);
4125   } else {
4126     breakpoint();
4127   }
4128   __ bind(ok);
4129 }
4130 #endif
4131 
4132 void LIR_Assembler::membar() {
4133   // QQQ sparc TSO uses this,
4134   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad));
4135 }
4136 
4137 void LIR_Assembler::membar_acquire() {
4138   // No x86 machines currently require load fences
4139 }
4140 
4141 void LIR_Assembler::membar_release() {
4142   // No x86 machines currently require store fences
4143 }
4144 
4145 void LIR_Assembler::membar_loadload() {
4146   // no-op
4147   //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
4148 }
4149 
4150 void LIR_Assembler::membar_storestore() {
4151   // no-op
4152   //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
4153 }
4154 
4155 void LIR_Assembler::membar_loadstore() {
4156   // no-op
4157   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
4158 }
4159 
4160 void LIR_Assembler::membar_storeload() {
4161   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
4162 }
4163 
4164 void LIR_Assembler::on_spin_wait() {
4165   __ pause ();
4166 }
4167 
4168 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
4169   assert(result_reg->is_register(), "check");
4170 #ifdef _LP64
4171   // __ get_thread(result_reg->as_register_lo());
4172   __ mov(result_reg->as_register(), r15_thread);
4173 #else
4174   __ get_thread(result_reg->as_register());
4175 #endif // _LP64
4176 }
4177 
4178 
4179 void LIR_Assembler::peephole(LIR_List*) {
4180   // do nothing for now
4181 }
4182 
4183 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
4184   assert(data == dest, "xchg/xadd uses only 2 operands");
4185 
4186   if (data->type() == T_INT) {
4187     if (code == lir_xadd) {
4188       __ lock();
4189       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
4190     } else {
4191       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
4192     }
4193   } else if (data->is_oop()) {
4194     assert (code == lir_xchg, "xadd for oops");
4195     Register obj = data->as_register();
4196 #ifdef _LP64
4197     if (UseCompressedOops) {
4198       __ encode_heap_oop(obj);
4199       __ xchgl(obj, as_Address(src->as_address_ptr()));
4200       __ decode_heap_oop(obj);
4201     } else {
4202       __ xchgptr(obj, as_Address(src->as_address_ptr()));
4203     }
4204 #else
4205     __ xchgl(obj, as_Address(src->as_address_ptr()));
4206 #endif
4207   } else if (data->type() == T_LONG) {
4208 #ifdef _LP64
4209     assert(data->as_register_lo() == data->as_register_hi(), "should be a single register");
4210     if (code == lir_xadd) {
4211       __ lock();
4212       __ xaddq(as_Address(src->as_address_ptr()), data->as_register_lo());
4213     } else {
4214       __ xchgq(data->as_register_lo(), as_Address(src->as_address_ptr()));
4215     }
4216 #else
4217     ShouldNotReachHere();
4218 #endif
4219   } else {
4220     ShouldNotReachHere();
4221   }
4222 }
4223 
4224 #undef __