1 /*
   2  * Copyright (c) 2000, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Compilation.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_MacroAssembler.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArrayKlass.hpp"
  36 #include "ci/ciInlineKlass.hpp"
  37 #include "ci/ciInstance.hpp"
  38 #include "code/compiledIC.hpp"
  39 #include "gc/shared/collectedHeap.hpp"
  40 #include "gc/shared/gc_globals.hpp"
  41 #include "nativeInst_aarch64.hpp"
  42 #include "oops/objArrayKlass.hpp"
  43 #include "oops/oop.inline.hpp"
  44 #include "runtime/frame.inline.hpp"
  45 #include "runtime/sharedRuntime.hpp"
  46 #include "runtime/stubRoutines.hpp"
  47 #include "utilities/powerOfTwo.hpp"
  48 #include "vmreg_aarch64.inline.hpp"
  49 
  50 
  51 #ifndef PRODUCT
  52 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  53 #else
  54 #define COMMENT(x)
  55 #endif
  56 
  57 NEEDS_CLEANUP // remove this definitions ?
  58 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  59 const Register SYNC_header = r0;   // synchronization header
  60 const Register SHIFT_count = r0;   // where count for shift operations must be
  61 
  62 #define __ _masm->
  63 
  64 
  65 static void select_different_registers(Register preserve,
  66                                        Register extra,
  67                                        Register &tmp1,
  68                                        Register &tmp2) {
  69   if (tmp1 == preserve) {
  70     assert_different_registers(tmp1, tmp2, extra);
  71     tmp1 = extra;
  72   } else if (tmp2 == preserve) {
  73     assert_different_registers(tmp1, tmp2, extra);
  74     tmp2 = extra;
  75   }
  76   assert_different_registers(preserve, tmp1, tmp2);
  77 }
  78 
  79 
  80 
  81 static void select_different_registers(Register preserve,
  82                                        Register extra,
  83                                        Register &tmp1,
  84                                        Register &tmp2,
  85                                        Register &tmp3) {
  86   if (tmp1 == preserve) {
  87     assert_different_registers(tmp1, tmp2, tmp3, extra);
  88     tmp1 = extra;
  89   } else if (tmp2 == preserve) {
  90     assert_different_registers(tmp1, tmp2, tmp3, extra);
  91     tmp2 = extra;
  92   } else if (tmp3 == preserve) {
  93     assert_different_registers(tmp1, tmp2, tmp3, extra);
  94     tmp3 = extra;
  95   }
  96   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  97 }
  98 
  99 
 100 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
 101 
 102 
 103 LIR_Opr LIR_Assembler::receiverOpr() {
 104   return FrameMap::receiver_opr;
 105 }
 106 
 107 LIR_Opr LIR_Assembler::osrBufferPointer() {
 108   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 109 }
 110 
 111 //--------------fpu register translations-----------------------
 112 
 113 
 114 address LIR_Assembler::float_constant(float f) {
 115   address const_addr = __ float_constant(f);
 116   if (const_addr == NULL) {
 117     bailout("const section overflow");
 118     return __ code()->consts()->start();
 119   } else {
 120     return const_addr;
 121   }
 122 }
 123 
 124 
 125 address LIR_Assembler::double_constant(double d) {
 126   address const_addr = __ double_constant(d);
 127   if (const_addr == NULL) {
 128     bailout("const section overflow");
 129     return __ code()->consts()->start();
 130   } else {
 131     return const_addr;
 132   }
 133 }
 134 
 135 address LIR_Assembler::int_constant(jlong n) {
 136   address const_addr = __ long_constant(n);
 137   if (const_addr == NULL) {
 138     bailout("const section overflow");
 139     return __ code()->consts()->start();
 140   } else {
 141     return const_addr;
 142   }
 143 }
 144 
 145 void LIR_Assembler::breakpoint() { Unimplemented(); }
 146 
 147 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 148 
 149 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 150 
 151 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
 152 //-------------------------------------------
 153 
 154 static Register as_reg(LIR_Opr op) {
 155   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 156 }
 157 
 158 static jlong as_long(LIR_Opr data) {
 159   jlong result;
 160   switch (data->type()) {
 161   case T_INT:
 162     result = (data->as_jint());
 163     break;
 164   case T_LONG:
 165     result = (data->as_jlong());
 166     break;
 167   default:
 168     ShouldNotReachHere();
 169     result = 0;  // unreachable
 170   }
 171   return result;
 172 }
 173 
 174 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 175   Register base = addr->base()->as_pointer_register();
 176   LIR_Opr opr = addr->index();
 177   if (opr->is_cpu_register()) {
 178     Register index;
 179     if (opr->is_single_cpu())
 180       index = opr->as_register();
 181     else
 182       index = opr->as_register_lo();
 183     assert(addr->disp() == 0, "must be");
 184     switch(opr->type()) {
 185       case T_INT:
 186         return Address(base, index, Address::sxtw(addr->scale()));
 187       case T_LONG:
 188         return Address(base, index, Address::lsl(addr->scale()));
 189       default:
 190         ShouldNotReachHere();
 191       }
 192   } else  {
 193     intptr_t addr_offset = intptr_t(addr->disp());
 194     if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
 195       return Address(base, addr_offset, Address::lsl(addr->scale()));
 196     else {
 197       __ mov(tmp, addr_offset);
 198       return Address(base, tmp, Address::lsl(addr->scale()));
 199     }
 200   }
 201   return Address();
 202 }
 203 
 204 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 205   ShouldNotReachHere();
 206   return Address();
 207 }
 208 
 209 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 210   return as_Address(addr, rscratch1);
 211 }
 212 
 213 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 214   return as_Address(addr, rscratch1);  // Ouch
 215   // FIXME: This needs to be much more clever.  See x86.
 216 }
 217 
 218 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
 219 // not encodable as a base + (immediate) offset, generate an explicit address
 220 // calculation to hold the address in a temporary register.
 221 Address LIR_Assembler::stack_slot_address(int index, uint size, Register tmp, int adjust) {
 222   precond(size == 4 || size == 8);
 223   Address addr = frame_map()->address_for_slot(index, adjust);
 224   precond(addr.getMode() == Address::base_plus_offset);
 225   precond(addr.base() == sp);
 226   precond(addr.offset() > 0);
 227   uint mask = size - 1;
 228   assert((addr.offset() & mask) == 0, "scaled offsets only");
 229   return __ legitimize_address(addr, size, tmp);
 230 }
 231 
 232 void LIR_Assembler::osr_entry() {
 233   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 234   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 235   ValueStack* entry_state = osr_entry->state();
 236   int number_of_locks = entry_state->locks_size();
 237 
 238   // we jump here if osr happens with the interpreter
 239   // state set up to continue at the beginning of the
 240   // loop that triggered osr - in particular, we have
 241   // the following registers setup:
 242   //
 243   // r2: osr buffer
 244   //
 245 
 246   // build frame
 247   ciMethod* m = compilation()->method();
 248   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 249 
 250   // OSR buffer is
 251   //
 252   // locals[nlocals-1..0]
 253   // monitors[0..number_of_locks]
 254   //
 255   // locals is a direct copy of the interpreter frame so in the osr buffer
 256   // so first slot in the local array is the last local from the interpreter
 257   // and last slot is local[0] (receiver) from the interpreter
 258   //
 259   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 260   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 261   // in the interpreter frame (the method lock if a sync method)
 262 
 263   // Initialize monitors in the compiled activation.
 264   //   r2: pointer to osr buffer
 265   //
 266   // All other registers are dead at this point and the locals will be
 267   // copied into place by code emitted in the IR.
 268 
 269   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 270   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 271     int monitor_offset = BytesPerWord * method()->max_locals() +
 272       (2 * BytesPerWord) * (number_of_locks - 1);
 273     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 274     // the OSR buffer using 2 word entries: first the lock and then
 275     // the oop.
 276     for (int i = 0; i < number_of_locks; i++) {
 277       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 278 #ifdef ASSERT
 279       // verify the interpreter's monitor has a non-null object
 280       {
 281         Label L;
 282         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 283         __ cbnz(rscratch1, L);
 284         __ stop("locked object is NULL");
 285         __ bind(L);
 286       }
 287 #endif
 288       __ ldr(r19, Address(OSR_buf, slot_offset + 0));
 289       __ str(r19, frame_map()->address_for_monitor_lock(i));
 290       __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 291       __ str(r19, frame_map()->address_for_monitor_object(i));
 292     }
 293   }
 294 }
 295 
 296 
 297 // inline cache check; done before the frame is built.
 298 int LIR_Assembler::check_icache() {
 299   Register receiver = FrameMap::receiver_opr->as_register();
 300   Register ic_klass = IC_Klass;
 301   int start_offset = __ offset();
 302   __ inline_cache_check(receiver, ic_klass);
 303 
 304   // if icache check fails, then jump to runtime routine
 305   // Note: RECEIVER must still contain the receiver!
 306   Label dont;
 307   __ br(Assembler::EQ, dont);
 308   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 309 
 310   // We align the verified entry point unless the method body
 311   // (including its inline cache check) will fit in a single 64-byte
 312   // icache line.
 313   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 314     // force alignment after the cache check.
 315     __ align(CodeEntryAlignment);
 316   }
 317 
 318   __ bind(dont);
 319   return start_offset;
 320 }
 321 
 322 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 323   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 324   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 325 
 326   Label L_skip_barrier;
 327 
 328   __ mov_metadata(rscratch2, method->holder()->constant_encoding());
 329   __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier /*L_fast_path*/);
 330   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 331   __ bind(L_skip_barrier);
 332 }
 333 
 334 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 335   if (o == NULL) {
 336     __ mov(reg, zr);
 337   } else {
 338     __ movoop(reg, o, /*immediate*/true);
 339   }
 340 }
 341 
 342 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 343   address target = NULL;
 344   relocInfo::relocType reloc_type = relocInfo::none;
 345 
 346   switch (patching_id(info)) {
 347   case PatchingStub::access_field_id:
 348     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 349     reloc_type = relocInfo::section_word_type;
 350     break;
 351   case PatchingStub::load_klass_id:
 352     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 353     reloc_type = relocInfo::metadata_type;
 354     break;
 355   case PatchingStub::load_mirror_id:
 356     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 357     reloc_type = relocInfo::oop_type;
 358     break;
 359   case PatchingStub::load_appendix_id:
 360     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 361     reloc_type = relocInfo::oop_type;
 362     break;
 363   default: ShouldNotReachHere();
 364   }
 365 
 366   __ far_call(RuntimeAddress(target));
 367   add_call_info_here(info);
 368 }
 369 
 370 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 371   deoptimize_trap(info);
 372 }
 373 
 374 
 375 // This specifies the rsp decrement needed to build the frame
 376 int LIR_Assembler::initial_frame_size_in_bytes() const {
 377   // if rounding, must let FrameMap know!
 378 
 379   return in_bytes(frame_map()->framesize_in_bytes());
 380 }
 381 
 382 
 383 int LIR_Assembler::emit_exception_handler() {
 384   // if the last instruction is a call (typically to do a throw which
 385   // is coming at the end after block reordering) the return address
 386   // must still point into the code area in order to avoid assertion
 387   // failures when searching for the corresponding bci => add a nop
 388   // (was bug 5/14/1999 - gri)
 389   __ nop();
 390 
 391   // generate code for exception handler
 392   address handler_base = __ start_a_stub(exception_handler_size());
 393   if (handler_base == NULL) {
 394     // not enough space left for the handler
 395     bailout("exception handler overflow");
 396     return -1;
 397   }
 398 
 399   int offset = code_offset();
 400 
 401   // the exception oop and pc are in r0, and r3
 402   // no other registers need to be preserved, so invalidate them
 403   __ invalidate_registers(false, true, true, false, true, true);
 404 
 405   // check that there is really an exception
 406   __ verify_not_null_oop(r0);
 407 
 408   // search an exception handler (r0: exception oop, r3: throwing pc)
 409   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));  __ should_not_reach_here();
 410   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 411   __ end_a_stub();
 412 
 413   return offset;
 414 }
 415 
 416 
 417 // Emit the code to remove the frame from the stack in the exception
 418 // unwind path.
 419 int LIR_Assembler::emit_unwind_handler() {
 420 #ifndef PRODUCT
 421   if (CommentedAssembly) {
 422     _masm->block_comment("Unwind handler");
 423   }
 424 #endif
 425 
 426   int offset = code_offset();
 427 
 428   // Fetch the exception from TLS and clear out exception related thread state
 429   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 430   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 431   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 432 
 433   __ bind(_unwind_handler_entry);
 434   __ verify_not_null_oop(r0);
 435   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 436     __ mov(r19, r0);  // Preserve the exception
 437   }
 438 
 439   // Preform needed unlocking
 440   MonitorExitStub* stub = NULL;
 441   if (method()->is_synchronized()) {
 442     monitor_address(0, FrameMap::r0_opr);
 443     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
 444     __ unlock_object(r5, r4, r0, *stub->entry());
 445     __ bind(*stub->continuation());
 446   }
 447 
 448   if (compilation()->env()->dtrace_method_probes()) {
 449     __ mov(c_rarg0, rthread);
 450     __ mov_metadata(c_rarg1, method()->constant_encoding());
 451     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 452   }
 453 
 454   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 455     __ mov(r0, r19);  // Restore the exception
 456   }
 457 
 458   // remove the activation and dispatch to the unwind handler
 459   __ block_comment("remove_frame and dispatch to the unwind handler");
 460   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 461   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 462 
 463   // Emit the slow path assembly
 464   if (stub != NULL) {
 465     stub->emit_code(this);
 466   }
 467 
 468   return offset;
 469 }
 470 
 471 
 472 int LIR_Assembler::emit_deopt_handler() {
 473   // if the last instruction is a call (typically to do a throw which
 474   // is coming at the end after block reordering) the return address
 475   // must still point into the code area in order to avoid assertion
 476   // failures when searching for the corresponding bci => add a nop
 477   // (was bug 5/14/1999 - gri)
 478   __ nop();
 479 
 480   // generate code for exception handler
 481   address handler_base = __ start_a_stub(deopt_handler_size());
 482   if (handler_base == NULL) {
 483     // not enough space left for the handler
 484     bailout("deopt handler overflow");
 485     return -1;
 486   }
 487 
 488   int offset = code_offset();
 489 
 490   __ adr(lr, pc());
 491   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 492   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 493   __ end_a_stub();
 494 
 495   return offset;
 496 }
 497 
 498 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 499   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 500   int pc_offset = code_offset();
 501   flush_debug_info(pc_offset);
 502   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 503   if (info->exception_handlers() != NULL) {
 504     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 505   }
 506 }
 507 
 508 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 509   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 510 
 511   ciMethod* method = compilation()->method();
 512   if (InlineTypeReturnedAsFields && method->signature()->returns_null_free_inline_type()) {
 513     ciInlineKlass* vk = method->return_type()->as_inline_klass();
 514     if (vk->can_be_returned_as_fields()) {
 515       address unpack_handler = vk->unpack_handler();
 516       assert(unpack_handler != NULL, "must be");
 517       __ far_call(RuntimeAddress(unpack_handler));
 518       // At this point, r0 points to the value object (for interpreter or C1 caller).
 519       // The fields of the object are copied into registers (for C2 caller).
 520     }
 521   }
 522 
 523   // Pop the stack before the safepoint code
 524   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 525 
 526   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 527     __ reserved_stack_check();
 528   }
 529 
 530   code_stub->set_safepoint_offset(__ offset());
 531   __ relocate(relocInfo::poll_return_type);
 532   __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
 533   __ ret(lr);
 534 }
 535 
 536 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
 537   return (__ store_inline_type_fields_to_buf(vk, false));
 538 }
 539 
 540 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 541   guarantee(info != NULL, "Shouldn't be NULL");
 542   __ get_polling_page(rscratch1, relocInfo::poll_type);
 543   add_debug_info_for_branch(info);  // This isn't just debug info:
 544                                     // it's the oop map
 545   __ read_polling_page(rscratch1, relocInfo::poll_type);
 546   return __ offset();
 547 }
 548 
 549 
 550 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 551   if (from_reg == r31_sp)
 552     from_reg = sp;
 553   if (to_reg == r31_sp)
 554     to_reg = sp;
 555   __ mov(to_reg, from_reg);
 556 }
 557 
 558 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 559 
 560 
 561 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 562   assert(src->is_constant(), "should not call otherwise");
 563   assert(dest->is_register(), "should not call otherwise");
 564   LIR_Const* c = src->as_constant_ptr();
 565 
 566   switch (c->type()) {
 567     case T_INT: {
 568       assert(patch_code == lir_patch_none, "no patching handled here");
 569       __ movw(dest->as_register(), c->as_jint());
 570       break;
 571     }
 572 
 573     case T_ADDRESS: {
 574       assert(patch_code == lir_patch_none, "no patching handled here");
 575       __ mov(dest->as_register(), c->as_jint());
 576       break;
 577     }
 578 
 579     case T_LONG: {
 580       assert(patch_code == lir_patch_none, "no patching handled here");
 581       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
 582       break;
 583     }
 584 
 585     case T_INLINE_TYPE:
 586     case T_OBJECT: {
 587         if (patch_code != lir_patch_none) {
 588           jobject2reg_with_patching(dest->as_register(), info);
 589         } else {
 590           jobject2reg(c->as_jobject(), dest->as_register());
 591         }
 592       break;
 593     }
 594 
 595     case T_METADATA: {
 596       if (patch_code != lir_patch_none) {
 597         klass2reg_with_patching(dest->as_register(), info);
 598       } else {
 599         __ mov_metadata(dest->as_register(), c->as_metadata());
 600       }
 601       break;
 602     }
 603 
 604     case T_FLOAT: {
 605       if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 606         __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
 607       } else {
 608         __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 609         __ ldrs(dest->as_float_reg(), Address(rscratch1));
 610       }
 611       break;
 612     }
 613 
 614     case T_DOUBLE: {
 615       if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
 616         __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
 617       } else {
 618         __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 619         __ ldrd(dest->as_double_reg(), Address(rscratch1));
 620       }
 621       break;
 622     }
 623 
 624     default:
 625       ShouldNotReachHere();
 626   }
 627 }
 628 
 629 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 630   LIR_Const* c = src->as_constant_ptr();
 631   switch (c->type()) {
 632   case T_INLINE_TYPE:
 633   case T_OBJECT:
 634     {
 635       if (! c->as_jobject())
 636         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 637       else {
 638         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 639         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 640       }
 641     }
 642     break;
 643   case T_ADDRESS:
 644     {
 645       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 646       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 647     }
 648   case T_INT:
 649   case T_FLOAT:
 650     {
 651       Register reg = zr;
 652       if (c->as_jint_bits() == 0)
 653         __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 654       else {
 655         __ movw(rscratch1, c->as_jint_bits());
 656         __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 657       }
 658     }
 659     break;
 660   case T_LONG:
 661   case T_DOUBLE:
 662     {
 663       Register reg = zr;
 664       if (c->as_jlong_bits() == 0)
 665         __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 666                                                  lo_word_offset_in_bytes));
 667       else {
 668         __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
 669         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 670                                                         lo_word_offset_in_bytes));
 671       }
 672     }
 673     break;
 674   default:
 675     ShouldNotReachHere();
 676   }
 677 }
 678 
 679 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 680   assert(src->is_constant(), "should not call otherwise");
 681   LIR_Const* c = src->as_constant_ptr();
 682   LIR_Address* to_addr = dest->as_address_ptr();
 683 
 684   void (Assembler::* insn)(Register Rt, const Address &adr);
 685 
 686   switch (type) {
 687   case T_ADDRESS:
 688     assert(c->as_jint() == 0, "should be");
 689     insn = &Assembler::str;
 690     break;
 691   case T_LONG:
 692     assert(c->as_jlong() == 0, "should be");
 693     insn = &Assembler::str;
 694     break;
 695   case T_INT:
 696     assert(c->as_jint() == 0, "should be");
 697     insn = &Assembler::strw;
 698     break;
 699   case T_INLINE_TYPE:
 700   case T_OBJECT:
 701   case T_ARRAY:
 702     // Non-null case is not handled on aarch64 but handled on x86
 703     // FIXME: do we need to add it here?
 704     assert(c->as_jobject() == 0, "should be");
 705     if (UseCompressedOops && !wide) {
 706       insn = &Assembler::strw;
 707     } else {
 708       insn = &Assembler::str;
 709     }
 710     break;
 711   case T_CHAR:
 712   case T_SHORT:
 713     assert(c->as_jint() == 0, "should be");
 714     insn = &Assembler::strh;
 715     break;
 716   case T_BOOLEAN:
 717   case T_BYTE:
 718     assert(c->as_jint() == 0, "should be");
 719     insn = &Assembler::strb;
 720     break;
 721   default:
 722     ShouldNotReachHere();
 723     insn = &Assembler::str;  // unreachable
 724   }
 725 
 726   if (info) add_debug_info_for_null_check_here(info);
 727   (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
 728 }
 729 
 730 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 731   assert(src->is_register(), "should not call otherwise");
 732   assert(dest->is_register(), "should not call otherwise");
 733 
 734   // move between cpu-registers
 735   if (dest->is_single_cpu()) {
 736     if (src->type() == T_LONG) {
 737       // Can do LONG -> OBJECT
 738       move_regs(src->as_register_lo(), dest->as_register());
 739       return;
 740     }
 741     assert(src->is_single_cpu(), "must match");
 742     if (src->type() == T_OBJECT || src->type() == T_INLINE_TYPE) {
 743       __ verify_oop(src->as_register());
 744     }
 745     move_regs(src->as_register(), dest->as_register());
 746 
 747   } else if (dest->is_double_cpu()) {
 748     if (is_reference_type(src->type())) {
 749       // Surprising to me but we can see move of a long to t_object
 750       __ verify_oop(src->as_register());
 751       move_regs(src->as_register(), dest->as_register_lo());
 752       return;
 753     }
 754     assert(src->is_double_cpu(), "must match");
 755     Register f_lo = src->as_register_lo();
 756     Register f_hi = src->as_register_hi();
 757     Register t_lo = dest->as_register_lo();
 758     Register t_hi = dest->as_register_hi();
 759     assert(f_hi == f_lo, "must be same");
 760     assert(t_hi == t_lo, "must be same");
 761     move_regs(f_lo, t_lo);
 762 
 763   } else if (dest->is_single_fpu()) {
 764     __ fmovs(dest->as_float_reg(), src->as_float_reg());
 765 
 766   } else if (dest->is_double_fpu()) {
 767     __ fmovd(dest->as_double_reg(), src->as_double_reg());
 768 
 769   } else {
 770     ShouldNotReachHere();
 771   }
 772 }
 773 
 774 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 775   precond(src->is_register() && dest->is_stack());
 776 
 777   uint const c_sz32 = sizeof(uint32_t);
 778   uint const c_sz64 = sizeof(uint64_t);
 779 
 780   if (src->is_single_cpu()) {
 781     int index = dest->single_stack_ix();
 782     if (is_reference_type(type)) {
 783       __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 784       __ verify_oop(src->as_register());
 785     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 786       __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 787     } else {
 788       __ strw(src->as_register(), stack_slot_address(index, c_sz32, rscratch1));
 789     }
 790 
 791   } else if (src->is_double_cpu()) {
 792     int index = dest->double_stack_ix();
 793     Address dest_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
 794     __ str(src->as_register_lo(), dest_addr_LO);
 795 
 796   } else if (src->is_single_fpu()) {
 797     int index = dest->single_stack_ix();
 798     __ strs(src->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
 799 
 800   } else if (src->is_double_fpu()) {
 801     int index = dest->double_stack_ix();
 802     __ strd(src->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
 803 
 804   } else {
 805     ShouldNotReachHere();
 806   }
 807 }
 808 
 809 
 810 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
 811   LIR_Address* to_addr = dest->as_address_ptr();
 812   PatchingStub* patch = NULL;
 813   Register compressed_src = rscratch1;
 814 
 815   if (patch_code != lir_patch_none) {
 816     deoptimize_trap(info);
 817     return;
 818   }
 819 
 820   if (is_reference_type(type)) {
 821     __ verify_oop(src->as_register());
 822 
 823     if (UseCompressedOops && !wide) {
 824       __ encode_heap_oop(compressed_src, src->as_register());
 825     } else {
 826       compressed_src = src->as_register();
 827     }
 828   }
 829 
 830   int null_check_here = code_offset();
 831   switch (type) {
 832     case T_FLOAT: {
 833       __ strs(src->as_float_reg(), as_Address(to_addr));
 834       break;
 835     }
 836 
 837     case T_DOUBLE: {
 838       __ strd(src->as_double_reg(), as_Address(to_addr));
 839       break;
 840     }
 841 
 842     case T_INLINE_TYPE: // fall through
 843     case T_ARRAY:   // fall through
 844     case T_OBJECT:  // fall through
 845       if (UseCompressedOops && !wide) {
 846         __ strw(compressed_src, as_Address(to_addr, rscratch2));
 847       } else {
 848          __ str(compressed_src, as_Address(to_addr));
 849       }
 850       break;
 851     case T_METADATA:
 852       // We get here to store a method pointer to the stack to pass to
 853       // a dtrace runtime call. This can't work on 64 bit with
 854       // compressed klass ptrs: T_METADATA can be a compressed klass
 855       // ptr or a 64 bit method pointer.
 856       ShouldNotReachHere();
 857       __ str(src->as_register(), as_Address(to_addr));
 858       break;
 859     case T_ADDRESS:
 860       __ str(src->as_register(), as_Address(to_addr));
 861       break;
 862     case T_INT:
 863       __ strw(src->as_register(), as_Address(to_addr));
 864       break;
 865 
 866     case T_LONG: {
 867       __ str(src->as_register_lo(), as_Address_lo(to_addr));
 868       break;
 869     }
 870 
 871     case T_BYTE:    // fall through
 872     case T_BOOLEAN: {
 873       __ strb(src->as_register(), as_Address(to_addr));
 874       break;
 875     }
 876 
 877     case T_CHAR:    // fall through
 878     case T_SHORT:
 879       __ strh(src->as_register(), as_Address(to_addr));
 880       break;
 881 
 882     default:
 883       ShouldNotReachHere();
 884   }
 885   if (info != NULL) {
 886     add_debug_info_for_null_check(null_check_here, info);
 887   }
 888 }
 889 
 890 
 891 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 892   precond(src->is_stack() && dest->is_register());
 893 
 894   uint const c_sz32 = sizeof(uint32_t);
 895   uint const c_sz64 = sizeof(uint64_t);
 896 
 897   if (dest->is_single_cpu()) {
 898     int index = src->single_stack_ix();
 899     if (is_reference_type(type)) {
 900       __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 901       __ verify_oop(dest->as_register());
 902     } else if (type == T_METADATA || type == T_ADDRESS) {
 903       __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 904     } else {
 905       __ ldrw(dest->as_register(), stack_slot_address(index, c_sz32, rscratch1));
 906     }
 907 
 908   } else if (dest->is_double_cpu()) {
 909     int index = src->double_stack_ix();
 910     Address src_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
 911     __ ldr(dest->as_register_lo(), src_addr_LO);
 912 
 913   } else if (dest->is_single_fpu()) {
 914     int index = src->single_stack_ix();
 915     __ ldrs(dest->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
 916 
 917   } else if (dest->is_double_fpu()) {
 918     int index = src->double_stack_ix();
 919     __ ldrd(dest->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
 920 
 921   } else {
 922     ShouldNotReachHere();
 923   }
 924 }
 925 
 926 
 927 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 928   address target = NULL;
 929   relocInfo::relocType reloc_type = relocInfo::none;
 930 
 931   switch (patching_id(info)) {
 932   case PatchingStub::access_field_id:
 933     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 934     reloc_type = relocInfo::section_word_type;
 935     break;
 936   case PatchingStub::load_klass_id:
 937     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 938     reloc_type = relocInfo::metadata_type;
 939     break;
 940   case PatchingStub::load_mirror_id:
 941     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 942     reloc_type = relocInfo::oop_type;
 943     break;
 944   case PatchingStub::load_appendix_id:
 945     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 946     reloc_type = relocInfo::oop_type;
 947     break;
 948   default: ShouldNotReachHere();
 949   }
 950 
 951   __ far_call(RuntimeAddress(target));
 952   add_call_info_here(info);
 953 }
 954 
 955 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 956 
 957   LIR_Opr temp;
 958   if (type == T_LONG || type == T_DOUBLE)
 959     temp = FrameMap::rscratch1_long_opr;
 960   else
 961     temp = FrameMap::rscratch1_opr;
 962 
 963   stack2reg(src, temp, src->type());
 964   reg2stack(temp, dest, dest->type(), false);
 965 }
 966 
 967 
 968 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 969   LIR_Address* addr = src->as_address_ptr();
 970   LIR_Address* from_addr = src->as_address_ptr();
 971 
 972   if (addr->base()->type() == T_OBJECT || addr->base()->type() == T_INLINE_TYPE) {
 973     __ verify_oop(addr->base()->as_pointer_register());
 974   }
 975 
 976   if (patch_code != lir_patch_none) {
 977     deoptimize_trap(info);
 978     return;
 979   }
 980 
 981   if (info != NULL) {
 982     add_debug_info_for_null_check_here(info);
 983   }
 984   int null_check_here = code_offset();
 985   switch (type) {
 986     case T_FLOAT: {
 987       __ ldrs(dest->as_float_reg(), as_Address(from_addr));
 988       break;
 989     }
 990 
 991     case T_DOUBLE: {
 992       __ ldrd(dest->as_double_reg(), as_Address(from_addr));
 993       break;
 994     }
 995 
 996     case T_INLINE_TYPE: // fall through
 997     case T_ARRAY:   // fall through
 998     case T_OBJECT:  // fall through
 999       if (UseCompressedOops && !wide) {
1000         __ ldrw(dest->as_register(), as_Address(from_addr));
1001       } else {
1002          __ ldr(dest->as_register(), as_Address(from_addr));
1003       }
1004       break;
1005     case T_METADATA:
1006       // We get here to store a method pointer to the stack to pass to
1007       // a dtrace runtime call. This can't work on 64 bit with
1008       // compressed klass ptrs: T_METADATA can be a compressed klass
1009       // ptr or a 64 bit method pointer.
1010       ShouldNotReachHere();
1011       __ ldr(dest->as_register(), as_Address(from_addr));
1012       break;
1013     case T_ADDRESS:
1014       // FIXME: OMG this is a horrible kludge.  Any offset from an
1015       // address that matches klass_offset_in_bytes() will be loaded
1016       // as a word, not a long.
1017       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1018         __ ldrw(dest->as_register(), as_Address(from_addr));
1019       } else {
1020         __ ldr(dest->as_register(), as_Address(from_addr));
1021       }
1022       break;
1023     case T_INT:
1024       __ ldrw(dest->as_register(), as_Address(from_addr));
1025       break;
1026 
1027     case T_LONG: {
1028       __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1029       break;
1030     }
1031 
1032     case T_BYTE:
1033       __ ldrsb(dest->as_register(), as_Address(from_addr));
1034       break;
1035     case T_BOOLEAN: {
1036       __ ldrb(dest->as_register(), as_Address(from_addr));
1037       break;
1038     }
1039 
1040     case T_CHAR:
1041       __ ldrh(dest->as_register(), as_Address(from_addr));
1042       break;
1043     case T_SHORT:
1044       __ ldrsh(dest->as_register(), as_Address(from_addr));
1045       break;
1046 
1047     default:
1048       ShouldNotReachHere();
1049   }
1050 
1051   if (is_reference_type(type)) {
1052     if (UseCompressedOops && !wide) {
1053       __ decode_heap_oop(dest->as_register());
1054     }
1055 
1056     if (!UseZGC) {
1057       // Load barrier has not yet been applied, so ZGC can't verify the oop here
1058       __ verify_oop(dest->as_register());
1059     }
1060   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1061     if (UseCompressedClassPointers) {
1062       __ decode_klass_not_null(dest->as_register());
1063     }
1064   }
1065 }
1066 
1067 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1068   assert(dst->is_cpu_register(), "must be");
1069   assert(dst->type() == src->type(), "must be");
1070 
1071   if (src->is_cpu_register()) {
1072     reg2reg(src, dst);
1073   } else if (src->is_stack()) {
1074     stack2reg(src, dst, dst->type());
1075   } else if (src->is_constant()) {
1076     const2reg(src, dst, lir_patch_none, NULL);
1077   } else {
1078     ShouldNotReachHere();
1079   }
1080 }
1081 
1082 int LIR_Assembler::array_element_size(BasicType type) const {
1083   int elem_size = type2aelembytes(type);
1084   return exact_log2(elem_size);
1085 }
1086 
1087 
1088 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1089   switch (op->code()) {
1090   case lir_idiv:
1091   case lir_irem:
1092     arithmetic_idiv(op->code(),
1093                     op->in_opr1(),
1094                     op->in_opr2(),
1095                     op->in_opr3(),
1096                     op->result_opr(),
1097                     op->info());
1098     break;
1099   case lir_fmad:
1100     __ fmaddd(op->result_opr()->as_double_reg(),
1101               op->in_opr1()->as_double_reg(),
1102               op->in_opr2()->as_double_reg(),
1103               op->in_opr3()->as_double_reg());
1104     break;
1105   case lir_fmaf:
1106     __ fmadds(op->result_opr()->as_float_reg(),
1107               op->in_opr1()->as_float_reg(),
1108               op->in_opr2()->as_float_reg(),
1109               op->in_opr3()->as_float_reg());
1110     break;
1111   default:      ShouldNotReachHere(); break;
1112   }
1113 }
1114 
1115 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1116 #ifdef ASSERT
1117   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1118   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1119   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1120 #endif
1121 
1122   if (op->cond() == lir_cond_always) {
1123     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1124     __ b(*(op->label()));
1125   } else {
1126     Assembler::Condition acond;
1127     if (op->code() == lir_cond_float_branch) {
1128       bool is_unordered = (op->ublock() == op->block());
1129       // Assembler::EQ does not permit unordered branches, so we add
1130       // another branch here.  Likewise, Assembler::NE does not permit
1131       // ordered branches.
1132       if ((is_unordered && op->cond() == lir_cond_equal)
1133           || (!is_unordered && op->cond() == lir_cond_notEqual))
1134         __ br(Assembler::VS, *(op->ublock()->label()));
1135       switch(op->cond()) {
1136       case lir_cond_equal:        acond = Assembler::EQ; break;
1137       case lir_cond_notEqual:     acond = Assembler::NE; break;
1138       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1139       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1140       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1141       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1142       default:                    ShouldNotReachHere();
1143         acond = Assembler::EQ;  // unreachable
1144       }
1145     } else {
1146       switch (op->cond()) {
1147         case lir_cond_equal:        acond = Assembler::EQ; break;
1148         case lir_cond_notEqual:     acond = Assembler::NE; break;
1149         case lir_cond_less:         acond = Assembler::LT; break;
1150         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1151         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1152         case lir_cond_greater:      acond = Assembler::GT; break;
1153         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1154         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1155         default:                    ShouldNotReachHere();
1156           acond = Assembler::EQ;  // unreachable
1157       }
1158     }
1159     __ br(acond,*(op->label()));
1160   }
1161 }
1162 
1163 
1164 
1165 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1166   LIR_Opr src  = op->in_opr();
1167   LIR_Opr dest = op->result_opr();
1168 
1169   switch (op->bytecode()) {
1170     case Bytecodes::_i2f:
1171       {
1172         __ scvtfws(dest->as_float_reg(), src->as_register());
1173         break;
1174       }
1175     case Bytecodes::_i2d:
1176       {
1177         __ scvtfwd(dest->as_double_reg(), src->as_register());
1178         break;
1179       }
1180     case Bytecodes::_l2d:
1181       {
1182         __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1183         break;
1184       }
1185     case Bytecodes::_l2f:
1186       {
1187         __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1188         break;
1189       }
1190     case Bytecodes::_f2d:
1191       {
1192         __ fcvts(dest->as_double_reg(), src->as_float_reg());
1193         break;
1194       }
1195     case Bytecodes::_d2f:
1196       {
1197         __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1198         break;
1199       }
1200     case Bytecodes::_i2c:
1201       {
1202         __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1203         break;
1204       }
1205     case Bytecodes::_i2l:
1206       {
1207         __ sxtw(dest->as_register_lo(), src->as_register());
1208         break;
1209       }
1210     case Bytecodes::_i2s:
1211       {
1212         __ sxth(dest->as_register(), src->as_register());
1213         break;
1214       }
1215     case Bytecodes::_i2b:
1216       {
1217         __ sxtb(dest->as_register(), src->as_register());
1218         break;
1219       }
1220     case Bytecodes::_l2i:
1221       {
1222         _masm->block_comment("FIXME: This could be a no-op");
1223         __ uxtw(dest->as_register(), src->as_register_lo());
1224         break;
1225       }
1226     case Bytecodes::_d2l:
1227       {
1228         __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1229         break;
1230       }
1231     case Bytecodes::_f2i:
1232       {
1233         __ fcvtzsw(dest->as_register(), src->as_float_reg());
1234         break;
1235       }
1236     case Bytecodes::_f2l:
1237       {
1238         __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1239         break;
1240       }
1241     case Bytecodes::_d2i:
1242       {
1243         __ fcvtzdw(dest->as_register(), src->as_double_reg());
1244         break;
1245       }
1246     default: ShouldNotReachHere();
1247   }
1248 }
1249 
1250 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1251   if (op->init_check()) {
1252     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1253                                InstanceKlass::init_state_offset()));
1254     __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1255     add_debug_info_for_null_check_here(op->stub()->info());
1256     __ br(Assembler::NE, *op->stub()->entry());
1257   }
1258   __ allocate_object(op->obj()->as_register(),
1259                      op->tmp1()->as_register(),
1260                      op->tmp2()->as_register(),
1261                      op->header_size(),
1262                      op->object_size(),
1263                      op->klass()->as_register(),
1264                      *op->stub()->entry());
1265   __ bind(*op->stub()->continuation());
1266 }
1267 
1268 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1269   Register len =  op->len()->as_register();
1270   __ uxtw(len, len);
1271 
1272   if (UseSlowPath || op->type() == T_INLINE_TYPE ||
1273       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1274       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1275     __ b(*op->stub()->entry());
1276   } else {
1277     Register tmp1 = op->tmp1()->as_register();
1278     Register tmp2 = op->tmp2()->as_register();
1279     Register tmp3 = op->tmp3()->as_register();
1280     if (len == tmp1) {
1281       tmp1 = tmp3;
1282     } else if (len == tmp2) {
1283       tmp2 = tmp3;
1284     } else if (len == tmp3) {
1285       // everything is ok
1286     } else {
1287       __ mov(tmp3, len);
1288     }
1289     __ allocate_array(op->obj()->as_register(),
1290                       len,
1291                       tmp1,
1292                       tmp2,
1293                       arrayOopDesc::header_size(op->type()),
1294                       array_element_size(op->type()),
1295                       op->klass()->as_register(),
1296                       *op->stub()->entry());
1297   }
1298   __ bind(*op->stub()->continuation());
1299 }
1300 
1301 void LIR_Assembler::type_profile_helper(Register mdo,
1302                                         ciMethodData *md, ciProfileData *data,
1303                                         Register recv, Label* update_done) {
1304   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1305     Label next_test;
1306     // See if the receiver is receiver[n].
1307     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1308     __ ldr(rscratch1, Address(rscratch2));
1309     __ cmp(recv, rscratch1);
1310     __ br(Assembler::NE, next_test);
1311     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1312     __ addptr(data_addr, DataLayout::counter_increment);
1313     __ b(*update_done);
1314     __ bind(next_test);
1315   }
1316 
1317   // Didn't find receiver; find next empty slot and fill it in
1318   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1319     Label next_test;
1320     __ lea(rscratch2,
1321            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1322     Address recv_addr(rscratch2);
1323     __ ldr(rscratch1, recv_addr);
1324     __ cbnz(rscratch1, next_test);
1325     __ str(recv, recv_addr);
1326     __ mov(rscratch1, DataLayout::counter_increment);
1327     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1328     __ str(rscratch1, Address(rscratch2));
1329     __ b(*update_done);
1330     __ bind(next_test);
1331   }
1332 }
1333 
1334 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1335   // we always need a stub for the failure case.
1336   CodeStub* stub = op->stub();
1337   Register obj = op->object()->as_register();
1338   Register k_RInfo = op->tmp1()->as_register();
1339   Register klass_RInfo = op->tmp2()->as_register();
1340   Register dst = op->result_opr()->as_register();
1341   ciKlass* k = op->klass();
1342   Register Rtmp1 = noreg;
1343 
1344   // check if it needs to be profiled
1345   ciMethodData* md;
1346   ciProfileData* data;
1347 
1348   const bool should_profile = op->should_profile();
1349 
1350   if (should_profile) {
1351     ciMethod* method = op->profiled_method();
1352     assert(method != NULL, "Should have method");
1353     int bci = op->profiled_bci();
1354     md = method->method_data_or_null();
1355     assert(md != NULL, "Sanity");
1356     data = md->bci_to_data(bci);
1357     assert(data != NULL,                "need data for type check");
1358     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1359   }
1360   Label profile_cast_success, profile_cast_failure;
1361   Label *success_target = should_profile ? &profile_cast_success : success;
1362   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1363 
1364   if (obj == k_RInfo) {
1365     k_RInfo = dst;
1366   } else if (obj == klass_RInfo) {
1367     klass_RInfo = dst;
1368   }
1369   if (k->is_loaded() && !UseCompressedClassPointers) {
1370     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1371   } else {
1372     Rtmp1 = op->tmp3()->as_register();
1373     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1374   }
1375 
1376   assert_different_registers(obj, k_RInfo, klass_RInfo);
1377 
1378   if (op->need_null_check()) {
1379     if (should_profile) {
1380       Label not_null;
1381       __ cbnz(obj, not_null);
1382       // Object is null; update MDO and exit
1383       Register mdo  = klass_RInfo;
1384       __ mov_metadata(mdo, md->constant_encoding());
1385       Address data_addr
1386         = __ form_address(rscratch2, mdo,
1387                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1388                           0);
1389       __ ldrb(rscratch1, data_addr);
1390       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1391       __ strb(rscratch1, data_addr);
1392       __ b(*obj_is_null);
1393       __ bind(not_null);
1394     } else {
1395       __ cbz(obj, *obj_is_null);
1396     }
1397   }
1398 
1399   if (!k->is_loaded()) {
1400     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1401   } else {
1402     __ mov_metadata(k_RInfo, k->constant_encoding());
1403   }
1404   __ verify_oop(obj);
1405 
1406   if (op->fast_check()) {
1407     // get object class
1408     // not a safepoint as obj null check happens earlier
1409     __ load_klass(rscratch1, obj);
1410     __ cmp( rscratch1, k_RInfo);
1411 
1412     __ br(Assembler::NE, *failure_target);
1413     // successful cast, fall through to profile or jump
1414   } else {
1415     // get object class
1416     // not a safepoint as obj null check happens earlier
1417     __ load_klass(klass_RInfo, obj);
1418     if (k->is_loaded()) {
1419       // See if we get an immediate positive hit
1420       __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1421       __ cmp(k_RInfo, rscratch1);
1422       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1423         __ br(Assembler::NE, *failure_target);
1424         // successful cast, fall through to profile or jump
1425       } else {
1426         // See if we get an immediate positive hit
1427         __ br(Assembler::EQ, *success_target);
1428         // check for self
1429         __ cmp(klass_RInfo, k_RInfo);
1430         __ br(Assembler::EQ, *success_target);
1431 
1432         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1433         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1434         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1435         // result is a boolean
1436         __ cbzw(klass_RInfo, *failure_target);
1437         // successful cast, fall through to profile or jump
1438       }
1439     } else {
1440       // perform the fast part of the checking logic
1441       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1442       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1443       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1444       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1445       __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1446       // result is a boolean
1447       __ cbz(k_RInfo, *failure_target);
1448       // successful cast, fall through to profile or jump
1449     }
1450   }
1451   if (should_profile) {
1452     Register mdo  = klass_RInfo, recv = k_RInfo;
1453     __ bind(profile_cast_success);
1454     __ mov_metadata(mdo, md->constant_encoding());
1455     __ load_klass(recv, obj);
1456     Label update_done;
1457     type_profile_helper(mdo, md, data, recv, success);
1458     __ b(*success);
1459 
1460     __ bind(profile_cast_failure);
1461     __ mov_metadata(mdo, md->constant_encoding());
1462     Address counter_addr
1463       = __ form_address(rscratch2, mdo,
1464                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1465                         0);
1466     __ ldr(rscratch1, counter_addr);
1467     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1468     __ str(rscratch1, counter_addr);
1469     __ b(*failure);
1470   }
1471   __ b(*success);
1472 }
1473 
1474 
1475 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1476   const bool should_profile = op->should_profile();
1477 
1478   LIR_Code code = op->code();
1479   if (code == lir_store_check) {
1480     Register value = op->object()->as_register();
1481     Register array = op->array()->as_register();
1482     Register k_RInfo = op->tmp1()->as_register();
1483     Register klass_RInfo = op->tmp2()->as_register();
1484     Register Rtmp1 = op->tmp3()->as_register();
1485 
1486     CodeStub* stub = op->stub();
1487 
1488     // check if it needs to be profiled
1489     ciMethodData* md;
1490     ciProfileData* data;
1491 
1492     if (should_profile) {
1493       ciMethod* method = op->profiled_method();
1494       assert(method != NULL, "Should have method");
1495       int bci = op->profiled_bci();
1496       md = method->method_data_or_null();
1497       assert(md != NULL, "Sanity");
1498       data = md->bci_to_data(bci);
1499       assert(data != NULL,                "need data for type check");
1500       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1501     }
1502     Label profile_cast_success, profile_cast_failure, done;
1503     Label *success_target = should_profile ? &profile_cast_success : &done;
1504     Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1505 
1506     if (should_profile) {
1507       Label not_null;
1508       __ cbnz(value, not_null);
1509       // Object is null; update MDO and exit
1510       Register mdo  = klass_RInfo;
1511       __ mov_metadata(mdo, md->constant_encoding());
1512       Address data_addr
1513         = __ form_address(rscratch2, mdo,
1514                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1515                           0);
1516       __ ldrb(rscratch1, data_addr);
1517       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1518       __ strb(rscratch1, data_addr);
1519       __ b(done);
1520       __ bind(not_null);
1521     } else {
1522       __ cbz(value, done);
1523     }
1524 
1525     add_debug_info_for_null_check_here(op->info_for_exception());
1526     __ load_klass(k_RInfo, array);
1527     __ load_klass(klass_RInfo, value);
1528 
1529     // get instance klass (it's already uncompressed)
1530     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1531     // perform the fast part of the checking logic
1532     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1533     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1534     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1535     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1536     __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1537     // result is a boolean
1538     __ cbzw(k_RInfo, *failure_target);
1539     // fall through to the success case
1540 
1541     if (should_profile) {
1542       Register mdo  = klass_RInfo, recv = k_RInfo;
1543       __ bind(profile_cast_success);
1544       __ mov_metadata(mdo, md->constant_encoding());
1545       __ load_klass(recv, value);
1546       Label update_done;
1547       type_profile_helper(mdo, md, data, recv, &done);
1548       __ b(done);
1549 
1550       __ bind(profile_cast_failure);
1551       __ mov_metadata(mdo, md->constant_encoding());
1552       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1553       __ lea(rscratch2, counter_addr);
1554       __ ldr(rscratch1, Address(rscratch2));
1555       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1556       __ str(rscratch1, Address(rscratch2));
1557       __ b(*stub->entry());
1558     }
1559 
1560     __ bind(done);
1561   } else if (code == lir_checkcast) {
1562     Register obj = op->object()->as_register();
1563     Register dst = op->result_opr()->as_register();
1564     Label success;
1565     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1566     __ bind(success);
1567     if (dst != obj) {
1568       __ mov(dst, obj);
1569     }
1570   } else if (code == lir_instanceof) {
1571     Register obj = op->object()->as_register();
1572     Register dst = op->result_opr()->as_register();
1573     Label success, failure, done;
1574     emit_typecheck_helper(op, &success, &failure, &failure);
1575     __ bind(failure);
1576     __ mov(dst, zr);
1577     __ b(done);
1578     __ bind(success);
1579     __ mov(dst, 1);
1580     __ bind(done);
1581   } else {
1582     ShouldNotReachHere();
1583   }
1584 }
1585 
1586 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1587   // We are loading/storing from/to an array that *may* be flattened (the
1588   // declared type is Object[], abstract[], interface[] or VT.ref[]).
1589   // If this array is flattened, take the slow path.
1590 
1591   Register klass = op->tmp()->as_register();
1592   if (UseArrayMarkWordCheck) {
1593     __ test_flattened_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1594   } else {
1595     __ load_klass(klass, op->array()->as_register());
1596     __ ldrw(klass, Address(klass, Klass::layout_helper_offset()));
1597     __ tst(klass, Klass::_lh_array_tag_vt_value_bit_inplace);
1598     __ br(Assembler::NE, *op->stub()->entry());
1599   }
1600   if (!op->value()->is_illegal()) {
1601     // The array is not flattened, but it might be null-free. If we are storing
1602     // a null into a null-free array, take the slow path (which will throw NPE).
1603     Label skip;
1604     __ cbnz(op->value()->as_register(), skip);
1605     if (UseArrayMarkWordCheck) {
1606       __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1607     } else {
1608       __ tst(klass, Klass::_lh_null_free_bit_inplace);
1609       __ br(Assembler::NE, *op->stub()->entry());
1610     }
1611     __ bind(skip);
1612   }
1613 }
1614 
1615 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1616   // We are storing into an array that *may* be null-free (the declared type is
1617   // Object[], abstract[], interface[] or VT.ref[]).
1618   if (UseArrayMarkWordCheck) {
1619     Label test_mark_word;
1620     Register tmp = op->tmp()->as_register();
1621     __ ldr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1622     __ tst(tmp, markWord::unlocked_value);
1623     __ br(Assembler::NE, test_mark_word);
1624     __ load_prototype_header(tmp, op->array()->as_register());
1625     __ bind(test_mark_word);
1626     __ tst(tmp, markWord::null_free_array_bit_in_place);
1627   } else {
1628     Register klass = op->tmp()->as_register();
1629     __ load_klass(klass, op->array()->as_register());
1630     __ ldr(klass, Address(klass, Klass::layout_helper_offset()));
1631     __ tst(klass, Klass::_lh_null_free_bit_inplace);
1632   }
1633 }
1634 
1635 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1636   Label L_oops_equal;
1637   Label L_oops_not_equal;
1638   Label L_end;
1639 
1640   Register left  = op->left()->as_register();
1641   Register right = op->right()->as_register();
1642 
1643   __ cmp(left, right);
1644   __ br(Assembler::EQ, L_oops_equal);
1645 
1646   // (1) Null check -- if one of the operands is null, the other must not be null (because
1647   //     the two references are not equal), so they are not substitutable,
1648   //     FIXME: do null check only if the operand is nullable
1649   {
1650     __ cbz(left, L_oops_not_equal);
1651     __ cbz(right, L_oops_not_equal);
1652   }
1653 
1654   ciKlass* left_klass = op->left_klass();
1655   ciKlass* right_klass = op->right_klass();
1656 
1657   // (2) Inline type check -- if either of the operands is not a inline type,
1658   //     they are not substitutable. We do this only if we are not sure that the
1659   //     operands are inline type
1660   if ((left_klass == NULL || right_klass == NULL) ||// The klass is still unloaded, or came from a Phi node.
1661       !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1662     Register tmp1  = op->tmp1()->as_register();
1663     __ mov(tmp1, markWord::inline_type_pattern);
1664     __ ldr(rscratch1, Address(left, oopDesc::mark_offset_in_bytes()));
1665     __ andr(tmp1, tmp1, rscratch1);
1666     __ ldr(rscratch1, Address(right, oopDesc::mark_offset_in_bytes()));
1667     __ andr(tmp1, tmp1, rscratch1);
1668     __ cmp(tmp1, (u1)markWord::inline_type_pattern);
1669     __ br(Assembler::NE, L_oops_not_equal);
1670   }
1671 
1672   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1673   if (left_klass != NULL && left_klass->is_inlinetype() && left_klass == right_klass) {
1674     // No need to load klass -- the operands are statically known to be the same inline klass.
1675     __ b(*op->stub()->entry());
1676   } else {
1677     Register left_klass_op = op->left_klass_op()->as_register();
1678     Register right_klass_op = op->right_klass_op()->as_register();
1679 
1680     if (UseCompressedClassPointers) {
1681       __ ldrw(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1682       __ ldrw(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1683       __ cmpw(left_klass_op, right_klass_op);
1684     } else {
1685       __ ldr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1686       __ ldr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1687       __ cmp(left_klass_op, right_klass_op);
1688     }
1689 
1690     __ br(Assembler::EQ, *op->stub()->entry()); // same klass -> do slow check
1691     // fall through to L_oops_not_equal
1692   }
1693 
1694   __ bind(L_oops_not_equal);
1695   move(op->not_equal_result(), op->result_opr());
1696   __ b(L_end);
1697 
1698   __ bind(L_oops_equal);
1699   move(op->equal_result(), op->result_opr());
1700   __ b(L_end);
1701 
1702   // We've returned from the stub. R0 contains 0x0 IFF the two
1703   // operands are not substitutable. (Don't compare against 0x1 in case the
1704   // C compiler is naughty)
1705   __ bind(*op->stub()->continuation());
1706   __ cbz(r0, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1707   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1708   // fall-through
1709   __ bind(L_end);
1710 }
1711 
1712 
1713 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1714   __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1715   __ cset(rscratch1, Assembler::NE);
1716   __ membar(__ AnyAny);
1717 }
1718 
1719 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1720   __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1721   __ cset(rscratch1, Assembler::NE);
1722   __ membar(__ AnyAny);
1723 }
1724 
1725 
1726 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1727   assert(VM_Version::supports_cx8(), "wrong machine");
1728   Register addr;
1729   if (op->addr()->is_register()) {
1730     addr = as_reg(op->addr());
1731   } else {
1732     assert(op->addr()->is_address(), "what else?");
1733     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1734     assert(addr_ptr->disp() == 0, "need 0 disp");
1735     assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index");
1736     addr = as_reg(addr_ptr->base());
1737   }
1738   Register newval = as_reg(op->new_value());
1739   Register cmpval = as_reg(op->cmp_value());
1740 
1741   if (op->code() == lir_cas_obj) {
1742     if (UseCompressedOops) {
1743       Register t1 = op->tmp1()->as_register();
1744       assert(op->tmp1()->is_valid(), "must be");
1745       __ encode_heap_oop(t1, cmpval);
1746       cmpval = t1;
1747       __ encode_heap_oop(rscratch2, newval);
1748       newval = rscratch2;
1749       casw(addr, newval, cmpval);
1750     } else {
1751       casl(addr, newval, cmpval);
1752     }
1753   } else if (op->code() == lir_cas_int) {
1754     casw(addr, newval, cmpval);
1755   } else {
1756     casl(addr, newval, cmpval);
1757   }
1758 }
1759 
1760 
1761 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1762 
1763   Assembler::Condition acond, ncond;
1764   switch (condition) {
1765   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1766   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1767   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1768   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1769   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1770   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1771   case lir_cond_belowEqual:
1772   case lir_cond_aboveEqual:
1773   default:                    ShouldNotReachHere();
1774     acond = Assembler::EQ; ncond = Assembler::NE;  // unreachable
1775   }
1776 
1777   assert(result->is_single_cpu() || result->is_double_cpu(),
1778          "expect single register for result");
1779   if (opr1->is_constant() && opr2->is_constant()
1780       && opr1->type() == T_INT && opr2->type() == T_INT) {
1781     jint val1 = opr1->as_jint();
1782     jint val2 = opr2->as_jint();
1783     if (val1 == 0 && val2 == 1) {
1784       __ cset(result->as_register(), ncond);
1785       return;
1786     } else if (val1 == 1 && val2 == 0) {
1787       __ cset(result->as_register(), acond);
1788       return;
1789     }
1790   }
1791 
1792   if (opr1->is_constant() && opr2->is_constant()
1793       && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1794     jlong val1 = opr1->as_jlong();
1795     jlong val2 = opr2->as_jlong();
1796     if (val1 == 0 && val2 == 1) {
1797       __ cset(result->as_register_lo(), ncond);
1798       return;
1799     } else if (val1 == 1 && val2 == 0) {
1800       __ cset(result->as_register_lo(), acond);
1801       return;
1802     }
1803   }
1804 
1805   if (opr1->is_stack()) {
1806     stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1807     opr1 = FrameMap::rscratch1_opr;
1808   } else if (opr1->is_constant()) {
1809     LIR_Opr tmp
1810       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1811     const2reg(opr1, tmp, lir_patch_none, NULL);
1812     opr1 = tmp;
1813   }
1814 
1815   if (opr2->is_stack()) {
1816     stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1817     opr2 = FrameMap::rscratch2_opr;
1818   } else if (opr2->is_constant()) {
1819     LIR_Opr tmp
1820       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1821     const2reg(opr2, tmp, lir_patch_none, NULL);
1822     opr2 = tmp;
1823   }
1824 
1825   if (result->type() == T_LONG)
1826     __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1827   else
1828     __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1829 }
1830 
1831 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1832   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1833 
1834   if (left->is_single_cpu()) {
1835     Register lreg = left->as_register();
1836     Register dreg = as_reg(dest);
1837 
1838     if (right->is_single_cpu()) {
1839       // cpu register - cpu register
1840 
1841       assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1842              "should be");
1843       Register rreg = right->as_register();
1844       switch (code) {
1845       case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1846       case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1847       case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1848       default:      ShouldNotReachHere();
1849       }
1850 
1851     } else if (right->is_double_cpu()) {
1852       Register rreg = right->as_register_lo();
1853       // single_cpu + double_cpu: can happen with obj+long
1854       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1855       switch (code) {
1856       case lir_add: __ add(dreg, lreg, rreg); break;
1857       case lir_sub: __ sub(dreg, lreg, rreg); break;
1858       default: ShouldNotReachHere();
1859       }
1860     } else if (right->is_constant()) {
1861       // cpu register - constant
1862       jlong c;
1863 
1864       // FIXME.  This is fugly: we really need to factor all this logic.
1865       switch(right->type()) {
1866       case T_LONG:
1867         c = right->as_constant_ptr()->as_jlong();
1868         break;
1869       case T_INT:
1870       case T_ADDRESS:
1871         c = right->as_constant_ptr()->as_jint();
1872         break;
1873       default:
1874         ShouldNotReachHere();
1875         c = 0;  // unreachable
1876         break;
1877       }
1878 
1879       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1880       if (c == 0 && dreg == lreg) {
1881         COMMENT("effective nop elided");
1882         return;
1883       }
1884       switch(left->type()) {
1885       case T_INT:
1886         switch (code) {
1887         case lir_add: __ addw(dreg, lreg, c); break;
1888         case lir_sub: __ subw(dreg, lreg, c); break;
1889         default: ShouldNotReachHere();
1890         }
1891         break;
1892       case T_OBJECT:
1893       case T_ADDRESS:
1894         switch (code) {
1895         case lir_add: __ add(dreg, lreg, c); break;
1896         case lir_sub: __ sub(dreg, lreg, c); break;
1897         default: ShouldNotReachHere();
1898         }
1899         break;
1900       default:
1901         ShouldNotReachHere();
1902       }
1903     } else {
1904       ShouldNotReachHere();
1905     }
1906 
1907   } else if (left->is_double_cpu()) {
1908     Register lreg_lo = left->as_register_lo();
1909 
1910     if (right->is_double_cpu()) {
1911       // cpu register - cpu register
1912       Register rreg_lo = right->as_register_lo();
1913       switch (code) {
1914       case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1915       case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1916       case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1917       case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1918       case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1919       default:
1920         ShouldNotReachHere();
1921       }
1922 
1923     } else if (right->is_constant()) {
1924       jlong c = right->as_constant_ptr()->as_jlong();
1925       Register dreg = as_reg(dest);
1926       switch (code) {
1927         case lir_add:
1928         case lir_sub:
1929           if (c == 0 && dreg == lreg_lo) {
1930             COMMENT("effective nop elided");
1931             return;
1932           }
1933           code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c);
1934           break;
1935         case lir_div:
1936           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1937           if (c == 1) {
1938             // move lreg_lo to dreg if divisor is 1
1939             __ mov(dreg, lreg_lo);
1940           } else {
1941             unsigned int shift = log2i_exact(c);
1942             // use rscratch1 as intermediate result register
1943             __ asr(rscratch1, lreg_lo, 63);
1944             __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift);
1945             __ asr(dreg, rscratch1, shift);
1946           }
1947           break;
1948         case lir_rem:
1949           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1950           if (c == 1) {
1951             // move 0 to dreg if divisor is 1
1952             __ mov(dreg, zr);
1953           } else {
1954             // use rscratch1 as intermediate result register
1955             __ negs(rscratch1, lreg_lo);
1956             __ andr(dreg, lreg_lo, c - 1);
1957             __ andr(rscratch1, rscratch1, c - 1);
1958             __ csneg(dreg, dreg, rscratch1, Assembler::MI);
1959           }
1960           break;
1961         default:
1962           ShouldNotReachHere();
1963       }
1964     } else {
1965       ShouldNotReachHere();
1966     }
1967   } else if (left->is_single_fpu()) {
1968     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1969     switch (code) {
1970     case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1971     case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1972     case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1973     case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1974     default:
1975       ShouldNotReachHere();
1976     }
1977   } else if (left->is_double_fpu()) {
1978     if (right->is_double_fpu()) {
1979       // fpu register - fpu register
1980       switch (code) {
1981       case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1982       case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1983       case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1984       case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1985       default:
1986         ShouldNotReachHere();
1987       }
1988     } else {
1989       if (right->is_constant()) {
1990         ShouldNotReachHere();
1991       }
1992       ShouldNotReachHere();
1993     }
1994   } else if (left->is_single_stack() || left->is_address()) {
1995     assert(left == dest, "left and dest must be equal");
1996     ShouldNotReachHere();
1997   } else {
1998     ShouldNotReachHere();
1999   }
2000 }
2001 
2002 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
2003 
2004 
2005 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
2006   switch(code) {
2007   case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
2008   case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
2009   default      : ShouldNotReachHere();
2010   }
2011 }
2012 
2013 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
2014 
2015   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
2016   Register Rleft = left->is_single_cpu() ? left->as_register() :
2017                                            left->as_register_lo();
2018    if (dst->is_single_cpu()) {
2019      Register Rdst = dst->as_register();
2020      if (right->is_constant()) {
2021        switch (code) {
2022          case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
2023          case lir_logic_or:  __ orrw (Rdst, Rleft, right->as_jint()); break;
2024          case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
2025          default: ShouldNotReachHere(); break;
2026        }
2027      } else {
2028        Register Rright = right->is_single_cpu() ? right->as_register() :
2029                                                   right->as_register_lo();
2030        switch (code) {
2031          case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
2032          case lir_logic_or:  __ orrw (Rdst, Rleft, Rright); break;
2033          case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
2034          default: ShouldNotReachHere(); break;
2035        }
2036      }
2037    } else {
2038      Register Rdst = dst->as_register_lo();
2039      if (right->is_constant()) {
2040        switch (code) {
2041          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
2042          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jlong()); break;
2043          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
2044          default: ShouldNotReachHere(); break;
2045        }
2046      } else {
2047        Register Rright = right->is_single_cpu() ? right->as_register() :
2048                                                   right->as_register_lo();
2049        switch (code) {
2050          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
2051          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
2052          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
2053          default: ShouldNotReachHere(); break;
2054        }
2055      }
2056    }
2057 }
2058 
2059 
2060 
2061 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) {
2062 
2063   // opcode check
2064   assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem");
2065   bool is_irem = (code == lir_irem);
2066 
2067   // operand check
2068   assert(left->is_single_cpu(),   "left must be register");
2069   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2070   assert(result->is_single_cpu(), "result must be register");
2071   Register lreg = left->as_register();
2072   Register dreg = result->as_register();
2073 
2074   // power-of-2 constant check and codegen
2075   if (right->is_constant()) {
2076     int c = right->as_constant_ptr()->as_jint();
2077     assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
2078     if (is_irem) {
2079       if (c == 1) {
2080         // move 0 to dreg if divisor is 1
2081         __ movw(dreg, zr);
2082       } else {
2083         // use rscratch1 as intermediate result register
2084         __ negsw(rscratch1, lreg);
2085         __ andw(dreg, lreg, c - 1);
2086         __ andw(rscratch1, rscratch1, c - 1);
2087         __ csnegw(dreg, dreg, rscratch1, Assembler::MI);
2088       }
2089     } else {
2090       if (c == 1) {
2091         // move lreg to dreg if divisor is 1
2092         __ movw(dreg, lreg);
2093       } else {
2094         unsigned int shift = exact_log2(c);
2095         // use rscratch1 as intermediate result register
2096         __ asrw(rscratch1, lreg, 31);
2097         __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift);
2098         __ asrw(dreg, rscratch1, shift);
2099       }
2100     }
2101   } else {
2102     Register rreg = right->as_register();
2103     __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1);
2104   }
2105 }
2106 
2107 
2108 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2109   if (opr1->is_constant() && opr2->is_single_cpu()) {
2110     // tableswitch
2111     Register reg = as_reg(opr2);
2112     struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
2113     __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
2114   } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
2115     Register reg1 = as_reg(opr1);
2116     if (opr2->is_single_cpu()) {
2117       // cpu register - cpu register
2118       Register reg2 = opr2->as_register();
2119       if (is_reference_type(opr1->type())) {
2120         __ cmpoop(reg1, reg2);
2121       } else {
2122         assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2123         __ cmpw(reg1, reg2);
2124       }
2125       return;
2126     }
2127     if (opr2->is_double_cpu()) {
2128       // cpu register - cpu register
2129       Register reg2 = opr2->as_register_lo();
2130       __ cmp(reg1, reg2);
2131       return;
2132     }
2133 
2134     if (opr2->is_constant()) {
2135       bool is_32bit = false; // width of register operand
2136       jlong imm;
2137 
2138       switch(opr2->type()) {
2139       case T_INT:
2140         imm = opr2->as_constant_ptr()->as_jint();
2141         is_32bit = true;
2142         break;
2143       case T_LONG:
2144         imm = opr2->as_constant_ptr()->as_jlong();
2145         break;
2146       case T_ADDRESS:
2147         imm = opr2->as_constant_ptr()->as_jint();
2148         break;
2149       case T_METADATA:
2150         imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
2151         break;
2152       case T_INLINE_TYPE:
2153       case T_OBJECT:
2154       case T_ARRAY:
2155         jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
2156         __ cmpoop(reg1, rscratch1);
2157         return;
2158       default:
2159         ShouldNotReachHere();
2160         imm = 0;  // unreachable
2161         break;
2162       }
2163 
2164       if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
2165         if (is_32bit)
2166           __ cmpw(reg1, imm);
2167         else
2168           __ subs(zr, reg1, imm);
2169         return;
2170       } else {
2171         __ mov(rscratch1, imm);
2172         if (is_32bit)
2173           __ cmpw(reg1, rscratch1);
2174         else
2175           __ cmp(reg1, rscratch1);
2176         return;
2177       }
2178     } else
2179       ShouldNotReachHere();
2180   } else if (opr1->is_single_fpu()) {
2181     FloatRegister reg1 = opr1->as_float_reg();
2182     assert(opr2->is_single_fpu(), "expect single float register");
2183     FloatRegister reg2 = opr2->as_float_reg();
2184     __ fcmps(reg1, reg2);
2185   } else if (opr1->is_double_fpu()) {
2186     FloatRegister reg1 = opr1->as_double_reg();
2187     assert(opr2->is_double_fpu(), "expect double float register");
2188     FloatRegister reg2 = opr2->as_double_reg();
2189     __ fcmpd(reg1, reg2);
2190   } else {
2191     ShouldNotReachHere();
2192   }
2193 }
2194 
2195 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2196   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2197     bool is_unordered_less = (code == lir_ucmp_fd2i);
2198     if (left->is_single_fpu()) {
2199       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2200     } else if (left->is_double_fpu()) {
2201       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2202     } else {
2203       ShouldNotReachHere();
2204     }
2205   } else if (code == lir_cmp_l2i) {
2206     Label done;
2207     __ cmp(left->as_register_lo(), right->as_register_lo());
2208     __ mov(dst->as_register(), (uint64_t)-1L);
2209     __ br(Assembler::LT, done);
2210     __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2211     __ bind(done);
2212   } else {
2213     ShouldNotReachHere();
2214   }
2215 }
2216 
2217 
2218 void LIR_Assembler::align_call(LIR_Code code) {  }
2219 
2220 
2221 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2222   address call = __ trampoline_call(Address(op->addr(), rtype));
2223   if (call == NULL) {
2224     bailout("trampoline stub overflow");
2225     return;
2226   }
2227   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2228 }
2229 
2230 
2231 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2232   address call = __ ic_call(op->addr());
2233   if (call == NULL) {
2234     bailout("trampoline stub overflow");
2235     return;
2236   }
2237   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2238 }
2239 
2240 void LIR_Assembler::emit_static_call_stub() {
2241   address call_pc = __ pc();
2242   address stub = __ start_a_stub(call_stub_size());
2243   if (stub == NULL) {
2244     bailout("static call stub overflow");
2245     return;
2246   }
2247 
2248   int start = __ offset();
2249 
2250   __ relocate(static_stub_Relocation::spec(call_pc));
2251   __ emit_static_call_stub();
2252 
2253   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
2254         <= call_stub_size(), "stub too big");
2255   __ end_a_stub();
2256 }
2257 
2258 
2259 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2260   assert(exceptionOop->as_register() == r0, "must match");
2261   assert(exceptionPC->as_register() == r3, "must match");
2262 
2263   // exception object is not added to oop map by LinearScan
2264   // (LinearScan assumes that no oops are in fixed registers)
2265   info->add_register_oop(exceptionOop);
2266   Runtime1::StubID unwind_id;
2267 
2268   // get current pc information
2269   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2270   if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
2271     // As no instructions have been generated yet for this LIR node it's
2272     // possible that an oop map already exists for the current offset.
2273     // In that case insert an dummy NOP here to ensure all oop map PCs
2274     // are unique. See JDK-8237483.
2275     __ nop();
2276   }
2277   int pc_for_athrow_offset = __ offset();
2278   InternalAddress pc_for_athrow(__ pc());
2279   __ adr(exceptionPC->as_register(), pc_for_athrow);
2280   add_call_info(pc_for_athrow_offset, info); // for exception handler
2281 
2282   __ verify_not_null_oop(r0);
2283   // search an exception handler (r0: exception oop, r3: throwing pc)
2284   if (compilation()->has_fpu_code()) {
2285     unwind_id = Runtime1::handle_exception_id;
2286   } else {
2287     unwind_id = Runtime1::handle_exception_nofpu_id;
2288   }
2289   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2290 
2291   // FIXME: enough room for two byte trap   ????
2292   __ nop();
2293 }
2294 
2295 
2296 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2297   assert(exceptionOop->as_register() == r0, "must match");
2298 
2299   __ b(_unwind_handler_entry);
2300 }
2301 
2302 
2303 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2304   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2305   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2306 
2307   switch (left->type()) {
2308     case T_INT: {
2309       switch (code) {
2310       case lir_shl:  __ lslvw (dreg, lreg, count->as_register()); break;
2311       case lir_shr:  __ asrvw (dreg, lreg, count->as_register()); break;
2312       case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2313       default:
2314         ShouldNotReachHere();
2315         break;
2316       }
2317       break;
2318     case T_LONG:
2319     case T_INLINE_TYPE:
2320     case T_ADDRESS:
2321     case T_OBJECT:
2322       switch (code) {
2323       case lir_shl:  __ lslv (dreg, lreg, count->as_register()); break;
2324       case lir_shr:  __ asrv (dreg, lreg, count->as_register()); break;
2325       case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2326       default:
2327         ShouldNotReachHere();
2328         break;
2329       }
2330       break;
2331     default:
2332       ShouldNotReachHere();
2333       break;
2334     }
2335   }
2336 }
2337 
2338 
2339 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2340   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2341   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2342 
2343   switch (left->type()) {
2344     case T_INT: {
2345       switch (code) {
2346       case lir_shl:  __ lslw (dreg, lreg, count); break;
2347       case lir_shr:  __ asrw (dreg, lreg, count); break;
2348       case lir_ushr: __ lsrw (dreg, lreg, count); break;
2349       default:
2350         ShouldNotReachHere();
2351         break;
2352       }
2353       break;
2354     case T_LONG:
2355     case T_ADDRESS:
2356     case T_INLINE_TYPE:
2357     case T_OBJECT:
2358       switch (code) {
2359       case lir_shl:  __ lsl (dreg, lreg, count); break;
2360       case lir_shr:  __ asr (dreg, lreg, count); break;
2361       case lir_ushr: __ lsr (dreg, lreg, count); break;
2362       default:
2363         ShouldNotReachHere();
2364         break;
2365       }
2366       break;
2367     default:
2368       ShouldNotReachHere();
2369       break;
2370     }
2371   }
2372 }
2373 
2374 
2375 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2376   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2377   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2378   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2379   __ str (r, Address(sp, offset_from_rsp_in_bytes));
2380 }
2381 
2382 
2383 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2384   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2385   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2386   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2387   __ mov (rscratch1, c);
2388   __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2389 }
2390 
2391 
2392 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2393   ShouldNotReachHere();
2394   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2395   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2396   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2397   __ lea(rscratch1, __ constant_oop_address(o));
2398   __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2399 }
2400 
2401 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2402   if (null_check) {
2403     __ cbz(obj, *slow_path->entry());
2404   }
2405   if (UseArrayMarkWordCheck) {
2406     if (is_dest) {
2407       __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2408     } else {
2409       __ test_flattened_array_oop(obj, tmp, *slow_path->entry());
2410     }
2411   } else {
2412     __ load_klass(tmp, obj);
2413     __ ldr(tmp, Address(tmp, Klass::layout_helper_offset()));
2414     if (is_dest) {
2415       // Take the slow path if it's a null_free destination array, in case the source array contains NULLs.
2416       __ tst(tmp, Klass::_lh_null_free_bit_inplace);
2417     } else {
2418       __ tst(tmp, Klass::_lh_array_tag_vt_value_bit_inplace);
2419     }
2420     __ br(Assembler::NE, *slow_path->entry());
2421   }
2422 }
2423 
2424 // This code replaces a call to arraycopy; no exception may
2425 // be thrown in this code, they must be thrown in the System.arraycopy
2426 // activation frame; we could save some checks if this would not be the case
2427 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2428   ciArrayKlass* default_type = op->expected_type();
2429   Register src = op->src()->as_register();
2430   Register dst = op->dst()->as_register();
2431   Register src_pos = op->src_pos()->as_register();
2432   Register dst_pos = op->dst_pos()->as_register();
2433   Register length  = op->length()->as_register();
2434   Register tmp = op->tmp()->as_register();
2435 
2436   CodeStub* stub = op->stub();
2437   int flags = op->flags();
2438   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2439   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2440 
2441   if (flags & LIR_OpArrayCopy::always_slow_path) {
2442     __ b(*stub->entry());
2443     __ bind(*stub->continuation());
2444     return;
2445   }
2446 
2447   // if we don't know anything, just go through the generic arraycopy
2448   if (default_type == NULL // || basic_type == T_OBJECT
2449       ) {
2450     Label done;
2451     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2452 
2453     // Save the arguments in case the generic arraycopy fails and we
2454     // have to fall back to the JNI stub
2455     __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2456     __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2457     __ str(src,              Address(sp, 4*BytesPerWord));
2458 
2459     address copyfunc_addr = StubRoutines::generic_arraycopy();
2460     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2461 
2462     // The arguments are in java calling convention so we shift them
2463     // to C convention
2464     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2465     __ mov(c_rarg0, j_rarg0);
2466     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2467     __ mov(c_rarg1, j_rarg1);
2468     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2469     __ mov(c_rarg2, j_rarg2);
2470     assert_different_registers(c_rarg3, j_rarg4);
2471     __ mov(c_rarg3, j_rarg3);
2472     __ mov(c_rarg4, j_rarg4);
2473 #ifndef PRODUCT
2474     if (PrintC1Statistics) {
2475       __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2476     }
2477 #endif
2478     __ far_call(RuntimeAddress(copyfunc_addr));
2479 
2480     __ cbz(r0, *stub->continuation());
2481 
2482     // Reload values from the stack so they are where the stub
2483     // expects them.
2484     __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2485     __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2486     __ ldr(src,              Address(sp, 4*BytesPerWord));
2487 
2488     // r0 is -1^K where K == partial copied count
2489     __ eonw(rscratch1, r0, zr);
2490     // adjust length down and src/end pos up by partial copied count
2491     __ subw(length, length, rscratch1);
2492     __ addw(src_pos, src_pos, rscratch1);
2493     __ addw(dst_pos, dst_pos, rscratch1);
2494     __ b(*stub->entry());
2495 
2496     __ bind(*stub->continuation());
2497     return;
2498   }
2499 
2500   // Handle inline type arrays
2501   if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2502     arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2503   }
2504 
2505   if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2506     arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2507   }
2508 
2509   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2510 
2511   int elem_size = type2aelembytes(basic_type);
2512   int scale = exact_log2(elem_size);
2513 
2514   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2515   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2516   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2517   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2518 
2519   // test for NULL
2520   if (flags & LIR_OpArrayCopy::src_null_check) {
2521     __ cbz(src, *stub->entry());
2522   }
2523   if (flags & LIR_OpArrayCopy::dst_null_check) {
2524     __ cbz(dst, *stub->entry());
2525   }
2526 
2527   // If the compiler was not able to prove that exact type of the source or the destination
2528   // of the arraycopy is an array type, check at runtime if the source or the destination is
2529   // an instance type.
2530   if (flags & LIR_OpArrayCopy::type_check) {
2531     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2532       __ load_klass(tmp, dst);
2533       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2534       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2535       __ br(Assembler::GE, *stub->entry());
2536     }
2537 
2538     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2539       __ load_klass(tmp, src);
2540       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2541       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2542       __ br(Assembler::GE, *stub->entry());
2543     }
2544   }
2545 
2546   // check if negative
2547   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2548     __ cmpw(src_pos, 0);
2549     __ br(Assembler::LT, *stub->entry());
2550   }
2551   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2552     __ cmpw(dst_pos, 0);
2553     __ br(Assembler::LT, *stub->entry());
2554   }
2555 
2556   if (flags & LIR_OpArrayCopy::length_positive_check) {
2557     __ cmpw(length, 0);
2558     __ br(Assembler::LT, *stub->entry());
2559   }
2560 
2561   if (flags & LIR_OpArrayCopy::src_range_check) {
2562     __ addw(tmp, src_pos, length);
2563     __ ldrw(rscratch1, src_length_addr);
2564     __ cmpw(tmp, rscratch1);
2565     __ br(Assembler::HI, *stub->entry());
2566   }
2567   if (flags & LIR_OpArrayCopy::dst_range_check) {
2568     __ addw(tmp, dst_pos, length);
2569     __ ldrw(rscratch1, dst_length_addr);
2570     __ cmpw(tmp, rscratch1);
2571     __ br(Assembler::HI, *stub->entry());
2572   }
2573 
2574   if (flags & LIR_OpArrayCopy::type_check) {
2575     // We don't know the array types are compatible
2576     if (basic_type != T_OBJECT) {
2577       // Simple test for basic type arrays
2578       if (UseCompressedClassPointers) {
2579         __ ldrw(tmp, src_klass_addr);
2580         __ ldrw(rscratch1, dst_klass_addr);
2581         __ cmpw(tmp, rscratch1);
2582       } else {
2583         __ ldr(tmp, src_klass_addr);
2584         __ ldr(rscratch1, dst_klass_addr);
2585         __ cmp(tmp, rscratch1);
2586       }
2587       __ br(Assembler::NE, *stub->entry());
2588     } else {
2589       // For object arrays, if src is a sub class of dst then we can
2590       // safely do the copy.
2591       Label cont, slow;
2592 
2593 #define PUSH(r1, r2)                                    \
2594       stp(r1, r2, __ pre(sp, -2 * wordSize));
2595 
2596 #define POP(r1, r2)                                     \
2597       ldp(r1, r2, __ post(sp, 2 * wordSize));
2598 
2599       __ PUSH(src, dst);
2600 
2601       __ load_klass(src, src);
2602       __ load_klass(dst, dst);
2603 
2604       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2605 
2606       __ PUSH(src, dst);
2607       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2608       __ POP(src, dst);
2609 
2610       __ cbnz(src, cont);
2611 
2612       __ bind(slow);
2613       __ POP(src, dst);
2614 
2615       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2616       if (copyfunc_addr != NULL) { // use stub if available
2617         // src is not a sub class of dst so we have to do a
2618         // per-element check.
2619 
2620         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2621         if ((flags & mask) != mask) {
2622           // Check that at least both of them object arrays.
2623           assert(flags & mask, "one of the two should be known to be an object array");
2624 
2625           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2626             __ load_klass(tmp, src);
2627           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2628             __ load_klass(tmp, dst);
2629           }
2630           int lh_offset = in_bytes(Klass::layout_helper_offset());
2631           Address klass_lh_addr(tmp, lh_offset);
2632           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2633           __ ldrw(rscratch1, klass_lh_addr);
2634           __ mov(rscratch2, objArray_lh);
2635           __ eorw(rscratch1, rscratch1, rscratch2);
2636           __ cbnzw(rscratch1, *stub->entry());
2637         }
2638 
2639        // Spill because stubs can use any register they like and it's
2640        // easier to restore just those that we care about.
2641         __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2642         __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2643         __ str(src,              Address(sp, 4*BytesPerWord));
2644 
2645         __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2646         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2647         assert_different_registers(c_rarg0, dst, dst_pos, length);
2648         __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2649         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2650         assert_different_registers(c_rarg1, dst, length);
2651         __ uxtw(c_rarg2, length);
2652         assert_different_registers(c_rarg2, dst);
2653 
2654         __ load_klass(c_rarg4, dst);
2655         __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2656         __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2657         __ far_call(RuntimeAddress(copyfunc_addr));
2658 
2659 #ifndef PRODUCT
2660         if (PrintC1Statistics) {
2661           Label failed;
2662           __ cbnz(r0, failed);
2663           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2664           __ bind(failed);
2665         }
2666 #endif
2667 
2668         __ cbz(r0, *stub->continuation());
2669 
2670 #ifndef PRODUCT
2671         if (PrintC1Statistics) {
2672           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2673         }
2674 #endif
2675         assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2676 
2677         // Restore previously spilled arguments
2678         __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2679         __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2680         __ ldr(src,              Address(sp, 4*BytesPerWord));
2681 
2682         // return value is -1^K where K is partial copied count
2683         __ eonw(rscratch1, r0, zr);
2684         // adjust length down and src/end pos up by partial copied count
2685         __ subw(length, length, rscratch1);
2686         __ addw(src_pos, src_pos, rscratch1);
2687         __ addw(dst_pos, dst_pos, rscratch1);
2688       }
2689 
2690       __ b(*stub->entry());
2691 
2692       __ bind(cont);
2693       __ POP(src, dst);
2694     }
2695   }
2696 
2697 #ifdef ASSERT
2698   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2699     // Sanity check the known type with the incoming class.  For the
2700     // primitive case the types must match exactly with src.klass and
2701     // dst.klass each exactly matching the default type.  For the
2702     // object array case, if no type check is needed then either the
2703     // dst type is exactly the expected type and the src type is a
2704     // subtype which we can't check or src is the same array as dst
2705     // but not necessarily exactly of type default_type.
2706     Label known_ok, halt;
2707     __ mov_metadata(tmp, default_type->constant_encoding());
2708     if (UseCompressedClassPointers) {
2709       __ encode_klass_not_null(tmp);
2710     }
2711 
2712     if (basic_type != T_OBJECT) {
2713 
2714       if (UseCompressedClassPointers) {
2715         __ ldrw(rscratch1, dst_klass_addr);
2716         __ cmpw(tmp, rscratch1);
2717       } else {
2718         __ ldr(rscratch1, dst_klass_addr);
2719         __ cmp(tmp, rscratch1);
2720       }
2721       __ br(Assembler::NE, halt);
2722       if (UseCompressedClassPointers) {
2723         __ ldrw(rscratch1, src_klass_addr);
2724         __ cmpw(tmp, rscratch1);
2725       } else {
2726         __ ldr(rscratch1, src_klass_addr);
2727         __ cmp(tmp, rscratch1);
2728       }
2729       __ br(Assembler::EQ, known_ok);
2730     } else {
2731       if (UseCompressedClassPointers) {
2732         __ ldrw(rscratch1, dst_klass_addr);
2733         __ cmpw(tmp, rscratch1);
2734       } else {
2735         __ ldr(rscratch1, dst_klass_addr);
2736         __ cmp(tmp, rscratch1);
2737       }
2738       __ br(Assembler::EQ, known_ok);
2739       __ cmp(src, dst);
2740       __ br(Assembler::EQ, known_ok);
2741     }
2742     __ bind(halt);
2743     __ stop("incorrect type information in arraycopy");
2744     __ bind(known_ok);
2745   }
2746 #endif
2747 
2748 #ifndef PRODUCT
2749   if (PrintC1Statistics) {
2750     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2751   }
2752 #endif
2753 
2754   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2755   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2756   assert_different_registers(c_rarg0, dst, dst_pos, length);
2757   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2758   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2759   assert_different_registers(c_rarg1, dst, length);
2760   __ uxtw(c_rarg2, length);
2761   assert_different_registers(c_rarg2, dst);
2762 
2763   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2764   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2765   const char *name;
2766   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2767 
2768  CodeBlob *cb = CodeCache::find_blob(entry);
2769  if (cb) {
2770    __ far_call(RuntimeAddress(entry));
2771  } else {
2772    __ call_VM_leaf(entry, 3);
2773  }
2774 
2775   __ bind(*stub->continuation());
2776 }
2777 
2778 
2779 
2780 
2781 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2782   Register obj = op->obj_opr()->as_register();  // may not be an oop
2783   Register hdr = op->hdr_opr()->as_register();
2784   Register lock = op->lock_opr()->as_register();
2785   if (!UseFastLocking) {
2786     __ b(*op->stub()->entry());
2787   } else if (op->code() == lir_lock) {
2788     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2789     // add debug info for NullPointerException only if one is possible
2790     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
2791     if (op->info() != NULL) {
2792       add_debug_info_for_null_check(null_check_offset, op->info());
2793     }
2794     // done
2795   } else if (op->code() == lir_unlock) {
2796     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2797     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2798   } else {
2799     Unimplemented();
2800   }
2801   __ bind(*op->stub()->continuation());
2802 }
2803 
2804 
2805 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2806   ciMethod* method = op->profiled_method();
2807   int bci          = op->profiled_bci();
2808   ciMethod* callee = op->profiled_callee();
2809 
2810   // Update counter for all call types
2811   ciMethodData* md = method->method_data_or_null();
2812   assert(md != NULL, "Sanity");
2813   ciProfileData* data = md->bci_to_data(bci);
2814   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2815   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2816   Register mdo  = op->mdo()->as_register();
2817   __ mov_metadata(mdo, md->constant_encoding());
2818   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2819   // Perform additional virtual call profiling for invokevirtual and
2820   // invokeinterface bytecodes
2821   if (op->should_profile_receiver_type()) {
2822     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2823     Register recv = op->recv()->as_register();
2824     assert_different_registers(mdo, recv);
2825     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2826     ciKlass* known_klass = op->known_holder();
2827     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2828       // We know the type that will be seen at this call site; we can
2829       // statically update the MethodData* rather than needing to do
2830       // dynamic tests on the receiver type
2831 
2832       // NOTE: we should probably put a lock around this search to
2833       // avoid collisions by concurrent compilations
2834       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2835       uint i;
2836       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2837         ciKlass* receiver = vc_data->receiver(i);
2838         if (known_klass->equals(receiver)) {
2839           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2840           __ addptr(data_addr, DataLayout::counter_increment);
2841           return;
2842         }
2843       }
2844 
2845       // Receiver type not found in profile data; select an empty slot
2846 
2847       // Note that this is less efficient than it should be because it
2848       // always does a write to the receiver part of the
2849       // VirtualCallData rather than just the first time
2850       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2851         ciKlass* receiver = vc_data->receiver(i);
2852         if (receiver == NULL) {
2853           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2854           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2855           __ lea(rscratch2, recv_addr);
2856           __ str(rscratch1, Address(rscratch2));
2857           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2858           __ addptr(data_addr, DataLayout::counter_increment);
2859           return;
2860         }
2861       }
2862     } else {
2863       __ load_klass(recv, recv);
2864       Label update_done;
2865       type_profile_helper(mdo, md, data, recv, &update_done);
2866       // Receiver did not match any saved receiver and there is no empty row for it.
2867       // Increment total counter to indicate polymorphic case.
2868       __ addptr(counter_addr, DataLayout::counter_increment);
2869 
2870       __ bind(update_done);
2871     }
2872   } else {
2873     // Static call
2874     __ addptr(counter_addr, DataLayout::counter_increment);
2875   }
2876 }
2877 
2878 
2879 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2880   Unimplemented();
2881 }
2882 
2883 
2884 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2885   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2886 }
2887 
2888 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2889   assert(op->crc()->is_single_cpu(),  "crc must be register");
2890   assert(op->val()->is_single_cpu(),  "byte value must be register");
2891   assert(op->result_opr()->is_single_cpu(), "result must be register");
2892   Register crc = op->crc()->as_register();
2893   Register val = op->val()->as_register();
2894   Register res = op->result_opr()->as_register();
2895 
2896   assert_different_registers(val, crc, res);
2897   uint64_t offset;
2898   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2899   if (offset) __ add(res, res, offset);
2900 
2901   __ mvnw(crc, crc); // ~crc
2902   __ update_byte_crc32(crc, val, res);
2903   __ mvnw(res, crc); // ~crc
2904 }
2905 
2906 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2907   COMMENT("emit_profile_type {");
2908   Register obj = op->obj()->as_register();
2909   Register tmp = op->tmp()->as_pointer_register();
2910   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2911   ciKlass* exact_klass = op->exact_klass();
2912   intptr_t current_klass = op->current_klass();
2913   bool not_null = op->not_null();
2914   bool no_conflict = op->no_conflict();
2915 
2916   Label update, next, none;
2917 
2918   bool do_null = !not_null;
2919   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2920   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2921 
2922   assert(do_null || do_update, "why are we here?");
2923   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2924   assert(mdo_addr.base() != rscratch1, "wrong register");
2925 
2926   __ verify_oop(obj);
2927 
2928   if (tmp != obj) {
2929     __ mov(tmp, obj);
2930   }
2931   if (do_null) {
2932     __ cbnz(tmp, update);
2933     if (!TypeEntries::was_null_seen(current_klass)) {
2934       __ ldr(rscratch2, mdo_addr);
2935       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2936       __ str(rscratch2, mdo_addr);
2937     }
2938     if (do_update) {
2939 #ifndef ASSERT
2940       __ b(next);
2941     }
2942 #else
2943       __ b(next);
2944     }
2945   } else {
2946     __ cbnz(tmp, update);
2947     __ stop("unexpected null obj");
2948 #endif
2949   }
2950 
2951   __ bind(update);
2952 
2953   if (do_update) {
2954 #ifdef ASSERT
2955     if (exact_klass != NULL) {
2956       Label ok;
2957       __ load_klass(tmp, tmp);
2958       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2959       __ eor(rscratch1, tmp, rscratch1);
2960       __ cbz(rscratch1, ok);
2961       __ stop("exact klass and actual klass differ");
2962       __ bind(ok);
2963     }
2964 #endif
2965     if (!no_conflict) {
2966       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2967         if (exact_klass != NULL) {
2968           __ mov_metadata(tmp, exact_klass->constant_encoding());
2969         } else {
2970           __ load_klass(tmp, tmp);
2971         }
2972 
2973         __ ldr(rscratch2, mdo_addr);
2974         __ eor(tmp, tmp, rscratch2);
2975         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2976         // klass seen before, nothing to do. The unknown bit may have been
2977         // set already but no need to check.
2978         __ cbz(rscratch1, next);
2979 
2980         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2981 
2982         if (TypeEntries::is_type_none(current_klass)) {
2983           __ cbz(rscratch2, none);
2984           __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2985           __ br(Assembler::EQ, none);
2986           // There is a chance that the checks above (re-reading profiling
2987           // data from memory) fail if another thread has just set the
2988           // profiling to this obj's klass
2989           __ dmb(Assembler::ISHLD);
2990           __ ldr(rscratch2, mdo_addr);
2991           __ eor(tmp, tmp, rscratch2);
2992           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2993           __ cbz(rscratch1, next);
2994         }
2995       } else {
2996         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2997                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2998 
2999         __ ldr(tmp, mdo_addr);
3000         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
3001       }
3002 
3003       // different than before. Cannot keep accurate profile.
3004       __ ldr(rscratch2, mdo_addr);
3005       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
3006       __ str(rscratch2, mdo_addr);
3007 
3008       if (TypeEntries::is_type_none(current_klass)) {
3009         __ b(next);
3010 
3011         __ bind(none);
3012         // first time here. Set profile type.
3013         __ str(tmp, mdo_addr);
3014       }
3015     } else {
3016       // There's a single possible klass at this profile point
3017       assert(exact_klass != NULL, "should be");
3018       if (TypeEntries::is_type_none(current_klass)) {
3019         __ mov_metadata(tmp, exact_klass->constant_encoding());
3020         __ ldr(rscratch2, mdo_addr);
3021         __ eor(tmp, tmp, rscratch2);
3022         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
3023         __ cbz(rscratch1, next);
3024 #ifdef ASSERT
3025         {
3026           Label ok;
3027           __ ldr(rscratch1, mdo_addr);
3028           __ cbz(rscratch1, ok);
3029           __ cmp(rscratch1, (u1)TypeEntries::null_seen);
3030           __ br(Assembler::EQ, ok);
3031           // may have been set by another thread
3032           __ dmb(Assembler::ISHLD);
3033           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
3034           __ ldr(rscratch2, mdo_addr);
3035           __ eor(rscratch2, rscratch1, rscratch2);
3036           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
3037           __ cbz(rscratch2, ok);
3038 
3039           __ stop("unexpected profiling mismatch");
3040           __ bind(ok);
3041         }
3042 #endif
3043         // first time here. Set profile type.
3044         __ str(tmp, mdo_addr);
3045       } else {
3046         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3047                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3048 
3049         __ ldr(tmp, mdo_addr);
3050         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
3051 
3052         __ orr(tmp, tmp, TypeEntries::type_unknown);
3053         __ str(tmp, mdo_addr);
3054         // FIXME: Write barrier needed here?
3055       }
3056     }
3057 
3058     __ bind(next);
3059   }
3060   COMMENT("} emit_profile_type");
3061 }
3062 
3063 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3064   Register obj = op->obj()->as_register();
3065   Register tmp = op->tmp()->as_pointer_register();
3066   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3067   bool not_null = op->not_null();
3068   int flag = op->flag();
3069 
3070   Label not_inline_type;
3071   if (!not_null) {
3072     __ cbz(obj, not_inline_type);
3073   }
3074 
3075   __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3076 
3077   __ ldrb(rscratch1, mdo_addr);
3078   __ orr(rscratch1, rscratch1, flag);
3079   __ strb(rscratch1, mdo_addr);
3080 
3081   __ bind(not_inline_type);
3082 }
3083 
3084 void LIR_Assembler::align_backward_branch_target() {
3085 }
3086 
3087 
3088 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3089   // tmp must be unused
3090   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
3091 
3092   if (left->is_single_cpu()) {
3093     assert(dest->is_single_cpu(), "expect single result reg");
3094     __ negw(dest->as_register(), left->as_register());
3095   } else if (left->is_double_cpu()) {
3096     assert(dest->is_double_cpu(), "expect double result reg");
3097     __ neg(dest->as_register_lo(), left->as_register_lo());
3098   } else if (left->is_single_fpu()) {
3099     assert(dest->is_single_fpu(), "expect single float result reg");
3100     __ fnegs(dest->as_float_reg(), left->as_float_reg());
3101   } else {
3102     assert(left->is_double_fpu(), "expect double float operand reg");
3103     assert(dest->is_double_fpu(), "expect double float result reg");
3104     __ fnegd(dest->as_double_reg(), left->as_double_reg());
3105   }
3106 }
3107 
3108 
3109 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3110   if (patch_code != lir_patch_none) {
3111     deoptimize_trap(info);
3112     return;
3113   }
3114 
3115   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
3116 }
3117 
3118 
3119 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3120   assert(!tmp->is_valid(), "don't need temporary");
3121 
3122   CodeBlob *cb = CodeCache::find_blob(dest);
3123   if (cb) {
3124     __ far_call(RuntimeAddress(dest));
3125   } else {
3126     __ mov(rscratch1, RuntimeAddress(dest));
3127     __ blr(rscratch1);
3128   }
3129 
3130   if (info != NULL) {
3131     add_call_info_here(info);
3132   }
3133 }
3134 
3135 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3136   if (dest->is_address() || src->is_address()) {
3137     move_op(src, dest, type, lir_patch_none, info,
3138             /*pop_fpu_stack*/false, /*wide*/false);
3139   } else {
3140     ShouldNotReachHere();
3141   }
3142 }
3143 
3144 #ifdef ASSERT
3145 // emit run-time assertion
3146 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3147   assert(op->code() == lir_assert, "must be");
3148 
3149   if (op->in_opr1()->is_valid()) {
3150     assert(op->in_opr2()->is_valid(), "both operands must be valid");
3151     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3152   } else {
3153     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3154     assert(op->condition() == lir_cond_always, "no other conditions allowed");
3155   }
3156 
3157   Label ok;
3158   if (op->condition() != lir_cond_always) {
3159     Assembler::Condition acond = Assembler::AL;
3160     switch (op->condition()) {
3161       case lir_cond_equal:        acond = Assembler::EQ;  break;
3162       case lir_cond_notEqual:     acond = Assembler::NE;  break;
3163       case lir_cond_less:         acond = Assembler::LT;  break;
3164       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
3165       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
3166       case lir_cond_greater:      acond = Assembler::GT;  break;
3167       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
3168       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
3169       default:                    ShouldNotReachHere();
3170     }
3171     __ br(acond, ok);
3172   }
3173   if (op->halt()) {
3174     const char* str = __ code_string(op->msg());
3175     __ stop(str);
3176   } else {
3177     breakpoint();
3178   }
3179   __ bind(ok);
3180 }
3181 #endif
3182 
3183 #ifndef PRODUCT
3184 #define COMMENT(x)   do { __ block_comment(x); } while (0)
3185 #else
3186 #define COMMENT(x)
3187 #endif
3188 
3189 void LIR_Assembler::membar() {
3190   COMMENT("membar");
3191   __ membar(MacroAssembler::AnyAny);
3192 }
3193 
3194 void LIR_Assembler::membar_acquire() {
3195   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3196 }
3197 
3198 void LIR_Assembler::membar_release() {
3199   __ membar(Assembler::LoadStore|Assembler::StoreStore);
3200 }
3201 
3202 void LIR_Assembler::membar_loadload() {
3203   __ membar(Assembler::LoadLoad);
3204 }
3205 
3206 void LIR_Assembler::membar_storestore() {
3207   __ membar(MacroAssembler::StoreStore);
3208 }
3209 
3210 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3211 
3212 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3213 
3214 void LIR_Assembler::on_spin_wait() {
3215   Unimplemented();
3216 }
3217 
3218 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3219   __ mov(result_reg->as_register(), rthread);
3220 }
3221 
3222 void LIR_Assembler::check_orig_pc() {
3223   __ ldr(rscratch2, frame_map()->address_for_orig_pc_addr());
3224   __ cmp(rscratch2, (u1)NULL_WORD);
3225 }
3226 
3227 void LIR_Assembler::peephole(LIR_List *lir) {
3228 #if 0
3229   if (tableswitch_count >= max_tableswitches)
3230     return;
3231 
3232   /*
3233     This finite-state automaton recognizes sequences of compare-and-
3234     branch instructions.  We will turn them into a tableswitch.  You
3235     could argue that C1 really shouldn't be doing this sort of
3236     optimization, but without it the code is really horrible.
3237   */
3238 
3239   enum { start_s, cmp1_s, beq_s, cmp_s } state;
3240   int first_key, last_key = -2147483648;
3241   int next_key = 0;
3242   int start_insn = -1;
3243   int last_insn = -1;
3244   Register reg = noreg;
3245   LIR_Opr reg_opr;
3246   state = start_s;
3247 
3248   LIR_OpList* inst = lir->instructions_list();
3249   for (int i = 0; i < inst->length(); i++) {
3250     LIR_Op* op = inst->at(i);
3251     switch (state) {
3252     case start_s:
3253       first_key = -1;
3254       start_insn = i;
3255       switch (op->code()) {
3256       case lir_cmp:
3257         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3258         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3259         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3260             && opr2->is_constant()
3261             && opr2->type() == T_INT) {
3262           reg_opr = opr1;
3263           reg = opr1->as_register();
3264           first_key = opr2->as_constant_ptr()->as_jint();
3265           next_key = first_key + 1;
3266           state = cmp_s;
3267           goto next_state;
3268         }
3269         break;
3270       }
3271       break;
3272     case cmp_s:
3273       switch (op->code()) {
3274       case lir_branch:
3275         if (op->as_OpBranch()->cond() == lir_cond_equal) {
3276           state = beq_s;
3277           last_insn = i;
3278           goto next_state;
3279         }
3280       }
3281       state = start_s;
3282       break;
3283     case beq_s:
3284       switch (op->code()) {
3285       case lir_cmp: {
3286         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3287         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3288         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3289             && opr1->as_register() == reg
3290             && opr2->is_constant()
3291             && opr2->type() == T_INT
3292             && opr2->as_constant_ptr()->as_jint() == next_key) {
3293           last_key = next_key;
3294           next_key++;
3295           state = cmp_s;
3296           goto next_state;
3297         }
3298       }
3299       }
3300       last_key = next_key;
3301       state = start_s;
3302       break;
3303     default:
3304       assert(false, "impossible state");
3305     }
3306     if (state == start_s) {
3307       if (first_key < last_key - 5L && reg != noreg) {
3308         {
3309           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3310           //        reg->encoding(),
3311           //        start_insn, first_key, last_key);
3312           //   for (int i = 0; i < inst->length(); i++) {
3313           //     inst->at(i)->print();
3314           //     tty->print("\n");
3315           //   }
3316           //   tty->print("\n");
3317         }
3318 
3319         struct tableswitch *sw = &switches[tableswitch_count];
3320         sw->_insn_index = start_insn, sw->_first_key = first_key,
3321           sw->_last_key = last_key, sw->_reg = reg;
3322         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3323         {
3324           // Insert the new table of branches
3325           int offset = last_insn;
3326           for (int n = first_key; n < last_key; n++) {
3327             inst->insert_before
3328               (last_insn + 1,
3329                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3330                                 inst->at(offset)->as_OpBranch()->label()));
3331             offset -= 2, i++;
3332           }
3333         }
3334         // Delete all the old compare-and-branch instructions
3335         for (int n = first_key; n < last_key; n++) {
3336           inst->remove_at(start_insn);
3337           inst->remove_at(start_insn);
3338         }
3339         // Insert the tableswitch instruction
3340         inst->insert_before(start_insn,
3341                             new LIR_Op2(lir_cmp, lir_cond_always,
3342                                         LIR_OprFact::intConst(tableswitch_count),
3343                                         reg_opr));
3344         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3345         tableswitch_count++;
3346       }
3347       reg = noreg;
3348       last_key = -2147483648;
3349     }
3350   next_state:
3351     ;
3352   }
3353 #endif
3354 }
3355 
3356 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3357   Address addr = as_Address(src->as_address_ptr());
3358   BasicType type = src->type();
3359   bool is_oop = is_reference_type(type);
3360 
3361   void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3362   void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3363 
3364   switch(type) {
3365   case T_INT:
3366     xchg = &MacroAssembler::atomic_xchgalw;
3367     add = &MacroAssembler::atomic_addalw;
3368     break;
3369   case T_LONG:
3370     xchg = &MacroAssembler::atomic_xchgal;
3371     add = &MacroAssembler::atomic_addal;
3372     break;
3373   case T_INLINE_TYPE:
3374   case T_OBJECT:
3375   case T_ARRAY:
3376     if (UseCompressedOops) {
3377       xchg = &MacroAssembler::atomic_xchgalw;
3378       add = &MacroAssembler::atomic_addalw;
3379     } else {
3380       xchg = &MacroAssembler::atomic_xchgal;
3381       add = &MacroAssembler::atomic_addal;
3382     }
3383     break;
3384   default:
3385     ShouldNotReachHere();
3386     xchg = &MacroAssembler::atomic_xchgal;
3387     add = &MacroAssembler::atomic_addal; // unreachable
3388   }
3389 
3390   switch (code) {
3391   case lir_xadd:
3392     {
3393       RegisterOrConstant inc;
3394       Register tmp = as_reg(tmp_op);
3395       Register dst = as_reg(dest);
3396       if (data->is_constant()) {
3397         inc = RegisterOrConstant(as_long(data));
3398         assert_different_registers(dst, addr.base(), tmp,
3399                                    rscratch1, rscratch2);
3400       } else {
3401         inc = RegisterOrConstant(as_reg(data));
3402         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3403                                    rscratch1, rscratch2);
3404       }
3405       __ lea(tmp, addr);
3406       (_masm->*add)(dst, inc, tmp);
3407       break;
3408     }
3409   case lir_xchg:
3410     {
3411       Register tmp = tmp_op->as_register();
3412       Register obj = as_reg(data);
3413       Register dst = as_reg(dest);
3414       if (is_oop && UseCompressedOops) {
3415         __ encode_heap_oop(rscratch2, obj);
3416         obj = rscratch2;
3417       }
3418       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3419       __ lea(tmp, addr);
3420       (_masm->*xchg)(dst, obj, tmp);
3421       if (is_oop && UseCompressedOops) {
3422         __ decode_heap_oop(dst);
3423       }
3424     }
3425     break;
3426   default:
3427     ShouldNotReachHere();
3428   }
3429   __ membar(__ AnyAny);
3430 }
3431 
3432 #undef __