1 /*
   2  * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Compilation.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_MacroAssembler.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArrayKlass.hpp"
  36 #include "ci/ciInlineKlass.hpp"
  37 #include "ci/ciInstance.hpp"
  38 #include "code/compiledIC.hpp"
  39 #include "gc/shared/collectedHeap.hpp"
  40 #include "gc/shared/gc_globals.hpp"
  41 #include "nativeInst_aarch64.hpp"
  42 #include "oops/objArrayKlass.hpp"
  43 #include "oops/oop.inline.hpp"
  44 #include "runtime/frame.inline.hpp"
  45 #include "runtime/sharedRuntime.hpp"
  46 #include "runtime/stubRoutines.hpp"
  47 #include "utilities/powerOfTwo.hpp"
  48 #include "vmreg_aarch64.inline.hpp"
  49 
  50 
  51 #ifndef PRODUCT
  52 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  53 #else
  54 #define COMMENT(x)
  55 #endif
  56 
  57 NEEDS_CLEANUP // remove this definitions ?
  58 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  59 const Register SYNC_header = r0;   // synchronization header
  60 const Register SHIFT_count = r0;   // where count for shift operations must be
  61 
  62 #define __ _masm->
  63 
  64 
  65 static void select_different_registers(Register preserve,
  66                                        Register extra,
  67                                        Register &tmp1,
  68                                        Register &tmp2) {
  69   if (tmp1 == preserve) {
  70     assert_different_registers(tmp1, tmp2, extra);
  71     tmp1 = extra;
  72   } else if (tmp2 == preserve) {
  73     assert_different_registers(tmp1, tmp2, extra);
  74     tmp2 = extra;
  75   }
  76   assert_different_registers(preserve, tmp1, tmp2);
  77 }
  78 
  79 
  80 
  81 static void select_different_registers(Register preserve,
  82                                        Register extra,
  83                                        Register &tmp1,
  84                                        Register &tmp2,
  85                                        Register &tmp3) {
  86   if (tmp1 == preserve) {
  87     assert_different_registers(tmp1, tmp2, tmp3, extra);
  88     tmp1 = extra;
  89   } else if (tmp2 == preserve) {
  90     assert_different_registers(tmp1, tmp2, tmp3, extra);
  91     tmp2 = extra;
  92   } else if (tmp3 == preserve) {
  93     assert_different_registers(tmp1, tmp2, tmp3, extra);
  94     tmp3 = extra;
  95   }
  96   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  97 }
  98 
  99 
 100 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
 101 
 102 
 103 LIR_Opr LIR_Assembler::receiverOpr() {
 104   return FrameMap::receiver_opr;
 105 }
 106 
 107 LIR_Opr LIR_Assembler::osrBufferPointer() {
 108   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 109 }
 110 
 111 //--------------fpu register translations-----------------------
 112 
 113 
 114 address LIR_Assembler::float_constant(float f) {
 115   address const_addr = __ float_constant(f);
 116   if (const_addr == NULL) {
 117     bailout("const section overflow");
 118     return __ code()->consts()->start();
 119   } else {
 120     return const_addr;
 121   }
 122 }
 123 
 124 
 125 address LIR_Assembler::double_constant(double d) {
 126   address const_addr = __ double_constant(d);
 127   if (const_addr == NULL) {
 128     bailout("const section overflow");
 129     return __ code()->consts()->start();
 130   } else {
 131     return const_addr;
 132   }
 133 }
 134 
 135 address LIR_Assembler::int_constant(jlong n) {
 136   address const_addr = __ long_constant(n);
 137   if (const_addr == NULL) {
 138     bailout("const section overflow");
 139     return __ code()->consts()->start();
 140   } else {
 141     return const_addr;
 142   }
 143 }
 144 
 145 void LIR_Assembler::breakpoint() { Unimplemented(); }
 146 
 147 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 148 
 149 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 150 
 151 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
 152 //-------------------------------------------
 153 
 154 static Register as_reg(LIR_Opr op) {
 155   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 156 }
 157 
 158 static jlong as_long(LIR_Opr data) {
 159   jlong result;
 160   switch (data->type()) {
 161   case T_INT:
 162     result = (data->as_jint());
 163     break;
 164   case T_LONG:
 165     result = (data->as_jlong());
 166     break;
 167   default:
 168     ShouldNotReachHere();
 169     result = 0;  // unreachable
 170   }
 171   return result;
 172 }
 173 
 174 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 175   Register base = addr->base()->as_pointer_register();
 176   LIR_Opr opr = addr->index();
 177   if (opr->is_cpu_register()) {
 178     Register index;
 179     if (opr->is_single_cpu())
 180       index = opr->as_register();
 181     else
 182       index = opr->as_register_lo();
 183     assert(addr->disp() == 0, "must be");
 184     switch(opr->type()) {
 185       case T_INT:
 186         return Address(base, index, Address::sxtw(addr->scale()));
 187       case T_LONG:
 188         return Address(base, index, Address::lsl(addr->scale()));
 189       default:
 190         ShouldNotReachHere();
 191       }
 192   } else {
 193     assert(addr->scale() == 0,
 194            "expected for immediate operand, was: %d", addr->scale());
 195     ptrdiff_t offset = ptrdiff_t(addr->disp());
 196     // NOTE: Does not handle any 16 byte vector access.
 197     const uint type_size = type2aelembytes(addr->type(), true);
 198     return __ legitimize_address(Address(base, offset), type_size, tmp);
 199   }
 200   return Address();
 201 }
 202 
 203 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 204   ShouldNotReachHere();
 205   return Address();
 206 }
 207 
 208 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 209   return as_Address(addr, rscratch1);
 210 }
 211 
 212 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 213   return as_Address(addr, rscratch1);  // Ouch
 214   // FIXME: This needs to be much more clever.  See x86.
 215 }
 216 
 217 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
 218 // not encodable as a base + (immediate) offset, generate an explicit address
 219 // calculation to hold the address in a temporary register.
 220 Address LIR_Assembler::stack_slot_address(int index, uint size, Register tmp, int adjust) {
 221   precond(size == 4 || size == 8);
 222   Address addr = frame_map()->address_for_slot(index, adjust);
 223   precond(addr.getMode() == Address::base_plus_offset);
 224   precond(addr.base() == sp);
 225   precond(addr.offset() > 0);
 226   uint mask = size - 1;
 227   assert((addr.offset() & mask) == 0, "scaled offsets only");
 228   return __ legitimize_address(addr, size, tmp);
 229 }
 230 
 231 void LIR_Assembler::osr_entry() {
 232   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 233   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 234   ValueStack* entry_state = osr_entry->state();
 235   int number_of_locks = entry_state->locks_size();
 236 
 237   // we jump here if osr happens with the interpreter
 238   // state set up to continue at the beginning of the
 239   // loop that triggered osr - in particular, we have
 240   // the following registers setup:
 241   //
 242   // r2: osr buffer
 243   //
 244 
 245   // build frame
 246   ciMethod* m = compilation()->method();
 247   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 248 
 249   // OSR buffer is
 250   //
 251   // locals[nlocals-1..0]
 252   // monitors[0..number_of_locks]
 253   //
 254   // locals is a direct copy of the interpreter frame so in the osr buffer
 255   // so first slot in the local array is the last local from the interpreter
 256   // and last slot is local[0] (receiver) from the interpreter
 257   //
 258   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 259   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 260   // in the interpreter frame (the method lock if a sync method)
 261 
 262   // Initialize monitors in the compiled activation.
 263   //   r2: pointer to osr buffer
 264   //
 265   // All other registers are dead at this point and the locals will be
 266   // copied into place by code emitted in the IR.
 267 
 268   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 269   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 270     int monitor_offset = BytesPerWord * method()->max_locals() +
 271       (2 * BytesPerWord) * (number_of_locks - 1);
 272     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 273     // the OSR buffer using 2 word entries: first the lock and then
 274     // the oop.
 275     for (int i = 0; i < number_of_locks; i++) {
 276       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 277 #ifdef ASSERT
 278       // verify the interpreter's monitor has a non-null object
 279       {
 280         Label L;
 281         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 282         __ cbnz(rscratch1, L);
 283         __ stop("locked object is NULL");
 284         __ bind(L);
 285       }
 286 #endif
 287       __ ldp(r19, r20, Address(OSR_buf, slot_offset));
 288       __ str(r19, frame_map()->address_for_monitor_lock(i));
 289       __ str(r20, frame_map()->address_for_monitor_object(i));
 290     }
 291   }
 292 }
 293 
 294 
 295 // inline cache check; done before the frame is built.
 296 int LIR_Assembler::check_icache() {
 297   Register receiver = FrameMap::receiver_opr->as_register();
 298   Register ic_klass = IC_Klass;
 299   int start_offset = __ offset();
 300   __ inline_cache_check(receiver, ic_klass);
 301 
 302   // if icache check fails, then jump to runtime routine
 303   // Note: RECEIVER must still contain the receiver!
 304   Label dont;
 305   __ br(Assembler::EQ, dont);
 306   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 307 
 308   // We align the verified entry point unless the method body
 309   // (including its inline cache check) will fit in a single 64-byte
 310   // icache line.
 311   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 312     // force alignment after the cache check.
 313     __ align(CodeEntryAlignment);
 314   }
 315 
 316   __ bind(dont);
 317   return start_offset;
 318 }
 319 
 320 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 321   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 322   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 323 
 324   Label L_skip_barrier;
 325 
 326   __ mov_metadata(rscratch2, method->holder()->constant_encoding());
 327   __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier /*L_fast_path*/);
 328   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 329   __ bind(L_skip_barrier);
 330 }
 331 
 332 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 333   if (o == NULL) {
 334     __ mov(reg, zr);
 335   } else {
 336     __ movoop(reg, o);
 337   }
 338 }
 339 
 340 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 341   address target = NULL;
 342   relocInfo::relocType reloc_type = relocInfo::none;
 343 
 344   switch (patching_id(info)) {
 345   case PatchingStub::access_field_id:
 346     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 347     reloc_type = relocInfo::section_word_type;
 348     break;
 349   case PatchingStub::load_klass_id:
 350     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 351     reloc_type = relocInfo::metadata_type;
 352     break;
 353   case PatchingStub::load_mirror_id:
 354     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 355     reloc_type = relocInfo::oop_type;
 356     break;
 357   case PatchingStub::load_appendix_id:
 358     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 359     reloc_type = relocInfo::oop_type;
 360     break;
 361   default: ShouldNotReachHere();
 362   }
 363 
 364   __ far_call(RuntimeAddress(target));
 365   add_call_info_here(info);
 366 }
 367 
 368 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 369   deoptimize_trap(info);
 370 }
 371 
 372 
 373 // This specifies the rsp decrement needed to build the frame
 374 int LIR_Assembler::initial_frame_size_in_bytes() const {
 375   // if rounding, must let FrameMap know!
 376 
 377   return in_bytes(frame_map()->framesize_in_bytes());
 378 }
 379 
 380 
 381 int LIR_Assembler::emit_exception_handler() {
 382   // generate code for exception handler
 383   address handler_base = __ start_a_stub(exception_handler_size());
 384   if (handler_base == NULL) {
 385     // not enough space left for the handler
 386     bailout("exception handler overflow");
 387     return -1;
 388   }
 389 
 390   int offset = code_offset();
 391 
 392   // the exception oop and pc are in r0, and r3
 393   // no other registers need to be preserved, so invalidate them
 394   __ invalidate_registers(false, true, true, false, true, true);
 395 
 396   // check that there is really an exception
 397   __ verify_not_null_oop(r0);
 398 
 399   // search an exception handler (r0: exception oop, r3: throwing pc)
 400   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 401   __ should_not_reach_here();
 402   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 403   __ end_a_stub();
 404 
 405   return offset;
 406 }
 407 
 408 
 409 // Emit the code to remove the frame from the stack in the exception
 410 // unwind path.
 411 int LIR_Assembler::emit_unwind_handler() {
 412 #ifndef PRODUCT
 413   if (CommentedAssembly) {
 414     _masm->block_comment("Unwind handler");
 415   }
 416 #endif
 417 
 418   int offset = code_offset();
 419 
 420   // Fetch the exception from TLS and clear out exception related thread state
 421   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 422   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 423   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 424 
 425   __ bind(_unwind_handler_entry);
 426   __ verify_not_null_oop(r0);
 427   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 428     __ mov(r19, r0);  // Preserve the exception
 429   }
 430 
 431   // Perform needed unlocking
 432   MonitorExitStub* stub = NULL;
 433   if (method()->is_synchronized()) {
 434     monitor_address(0, FrameMap::r0_opr);
 435     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
 436     if (UseHeavyMonitors) {
 437       __ b(*stub->entry());
 438     } else {
 439       __ unlock_object(r5, r4, r0, *stub->entry());
 440     }
 441     __ bind(*stub->continuation());
 442   }
 443 
 444   if (compilation()->env()->dtrace_method_probes()) {
 445     __ mov(c_rarg0, rthread);
 446     __ mov_metadata(c_rarg1, method()->constant_encoding());
 447     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 448   }
 449 
 450   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 451     __ mov(r0, r19);  // Restore the exception
 452   }
 453 
 454   // remove the activation and dispatch to the unwind handler
 455   __ block_comment("remove_frame and dispatch to the unwind handler");
 456   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 457   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 458 
 459   // Emit the slow path assembly
 460   if (stub != NULL) {
 461     stub->emit_code(this);
 462   }
 463 
 464   return offset;
 465 }
 466 
 467 
 468 int LIR_Assembler::emit_deopt_handler() {
 469   // generate code for exception handler
 470   address handler_base = __ start_a_stub(deopt_handler_size());
 471   if (handler_base == NULL) {
 472     // not enough space left for the handler
 473     bailout("deopt handler overflow");
 474     return -1;
 475   }
 476 
 477   int offset = code_offset();
 478 
 479   __ adr(lr, pc());
 480   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 481   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 482   __ end_a_stub();
 483 
 484   return offset;
 485 }
 486 
 487 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 488   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 489   int pc_offset = code_offset();
 490   flush_debug_info(pc_offset);
 491   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 492   if (info->exception_handlers() != NULL) {
 493     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 494   }
 495 }
 496 
 497 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 498   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 499 
 500   ciMethod* method = compilation()->method();
 501   if (InlineTypeReturnedAsFields && method->return_type()->is_inlinetype()) {
 502     ciInlineKlass* vk = method->return_type()->as_inline_klass();
 503     if (vk->can_be_returned_as_fields()) {
 504       address unpack_handler = vk->unpack_handler();
 505       assert(unpack_handler != NULL, "must be");
 506       __ far_call(RuntimeAddress(unpack_handler));
 507       // At this point, r0 points to the value object (for interpreter or C1 caller).
 508       // The fields of the object are copied into registers (for C2 caller).
 509     }
 510   }
 511 
 512   // Pop the stack before the safepoint code
 513   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 514 
 515   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 516     __ reserved_stack_check();
 517   }
 518 
 519   code_stub->set_safepoint_offset(__ offset());
 520   __ relocate(relocInfo::poll_return_type);
 521   __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
 522   __ ret(lr);
 523 }
 524 
 525 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
 526   return (__ store_inline_type_fields_to_buf(vk, false));
 527 }
 528 
 529 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 530   guarantee(info != NULL, "Shouldn't be NULL");
 531   __ get_polling_page(rscratch1, relocInfo::poll_type);
 532   add_debug_info_for_branch(info);  // This isn't just debug info:
 533                                     // it's the oop map
 534   __ read_polling_page(rscratch1, relocInfo::poll_type);
 535   return __ offset();
 536 }
 537 
 538 
 539 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 540   if (from_reg == r31_sp)
 541     from_reg = sp;
 542   if (to_reg == r31_sp)
 543     to_reg = sp;
 544   __ mov(to_reg, from_reg);
 545 }
 546 
 547 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 548 
 549 
 550 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 551   assert(src->is_constant(), "should not call otherwise");
 552   assert(dest->is_register(), "should not call otherwise");
 553   LIR_Const* c = src->as_constant_ptr();
 554 
 555   switch (c->type()) {
 556     case T_INT: {
 557       assert(patch_code == lir_patch_none, "no patching handled here");
 558       __ movw(dest->as_register(), c->as_jint());
 559       break;
 560     }
 561 
 562     case T_ADDRESS: {
 563       assert(patch_code == lir_patch_none, "no patching handled here");
 564       __ mov(dest->as_register(), c->as_jint());
 565       break;
 566     }
 567 
 568     case T_LONG: {
 569       assert(patch_code == lir_patch_none, "no patching handled here");
 570       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
 571       break;
 572     }
 573 
 574     case T_PRIMITIVE_OBJECT:
 575     case T_OBJECT: {
 576         if (patch_code != lir_patch_none) {
 577           jobject2reg_with_patching(dest->as_register(), info);
 578         } else {
 579           jobject2reg(c->as_jobject(), dest->as_register());
 580         }
 581       break;
 582     }
 583 
 584     case T_METADATA: {
 585       if (patch_code != lir_patch_none) {
 586         klass2reg_with_patching(dest->as_register(), info);
 587       } else {
 588         __ mov_metadata(dest->as_register(), c->as_metadata());
 589       }
 590       break;
 591     }
 592 
 593     case T_FLOAT: {
 594       if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 595         __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
 596       } else {
 597         __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 598         __ ldrs(dest->as_float_reg(), Address(rscratch1));
 599       }
 600       break;
 601     }
 602 
 603     case T_DOUBLE: {
 604       if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
 605         __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
 606       } else {
 607         __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 608         __ ldrd(dest->as_double_reg(), Address(rscratch1));
 609       }
 610       break;
 611     }
 612 
 613     default:
 614       ShouldNotReachHere();
 615   }
 616 }
 617 
 618 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 619   LIR_Const* c = src->as_constant_ptr();
 620   switch (c->type()) {
 621   case T_PRIMITIVE_OBJECT:
 622   case T_OBJECT:
 623     {
 624       if (! c->as_jobject())
 625         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 626       else {
 627         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 628         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 629       }
 630     }
 631     break;
 632   case T_ADDRESS:
 633     {
 634       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 635       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 636     }
 637   case T_INT:
 638   case T_FLOAT:
 639     {
 640       Register reg = zr;
 641       if (c->as_jint_bits() == 0)
 642         __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 643       else {
 644         __ movw(rscratch1, c->as_jint_bits());
 645         __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 646       }
 647     }
 648     break;
 649   case T_LONG:
 650   case T_DOUBLE:
 651     {
 652       Register reg = zr;
 653       if (c->as_jlong_bits() == 0)
 654         __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 655                                                  lo_word_offset_in_bytes));
 656       else {
 657         __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
 658         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 659                                                         lo_word_offset_in_bytes));
 660       }
 661     }
 662     break;
 663   default:
 664     ShouldNotReachHere();
 665   }
 666 }
 667 
 668 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 669   assert(src->is_constant(), "should not call otherwise");
 670   LIR_Const* c = src->as_constant_ptr();
 671   LIR_Address* to_addr = dest->as_address_ptr();
 672 
 673   void (Assembler::* insn)(Register Rt, const Address &adr);
 674 
 675   switch (type) {
 676   case T_ADDRESS:
 677     assert(c->as_jint() == 0, "should be");
 678     insn = &Assembler::str;
 679     break;
 680   case T_LONG:
 681     assert(c->as_jlong() == 0, "should be");
 682     insn = &Assembler::str;
 683     break;
 684   case T_INT:
 685     assert(c->as_jint() == 0, "should be");
 686     insn = &Assembler::strw;
 687     break;
 688   case T_PRIMITIVE_OBJECT:
 689   case T_OBJECT:
 690   case T_ARRAY:
 691     // Non-null case is not handled on aarch64 but handled on x86
 692     // FIXME: do we need to add it here?
 693     assert(c->as_jobject() == 0, "should be");
 694     if (UseCompressedOops && !wide) {
 695       insn = &Assembler::strw;
 696     } else {
 697       insn = &Assembler::str;
 698     }
 699     break;
 700   case T_CHAR:
 701   case T_SHORT:
 702     assert(c->as_jint() == 0, "should be");
 703     insn = &Assembler::strh;
 704     break;
 705   case T_BOOLEAN:
 706   case T_BYTE:
 707     assert(c->as_jint() == 0, "should be");
 708     insn = &Assembler::strb;
 709     break;
 710   default:
 711     ShouldNotReachHere();
 712     insn = &Assembler::str;  // unreachable
 713   }
 714 
 715   if (info) add_debug_info_for_null_check_here(info);
 716   (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
 717 }
 718 
 719 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 720   assert(src->is_register(), "should not call otherwise");
 721   assert(dest->is_register(), "should not call otherwise");
 722 
 723   // move between cpu-registers
 724   if (dest->is_single_cpu()) {
 725     if (src->type() == T_LONG) {
 726       // Can do LONG -> OBJECT
 727       move_regs(src->as_register_lo(), dest->as_register());
 728       return;
 729     }
 730     assert(src->is_single_cpu(), "must match");
 731     if (src->type() == T_OBJECT || src->type() == T_PRIMITIVE_OBJECT) {
 732       __ verify_oop(src->as_register());
 733     }
 734     move_regs(src->as_register(), dest->as_register());
 735 
 736   } else if (dest->is_double_cpu()) {
 737     if (is_reference_type(src->type())) {
 738       // Surprising to me but we can see move of a long to t_object
 739       __ verify_oop(src->as_register());
 740       move_regs(src->as_register(), dest->as_register_lo());
 741       return;
 742     }
 743     assert(src->is_double_cpu(), "must match");
 744     Register f_lo = src->as_register_lo();
 745     Register f_hi = src->as_register_hi();
 746     Register t_lo = dest->as_register_lo();
 747     Register t_hi = dest->as_register_hi();
 748     assert(f_hi == f_lo, "must be same");
 749     assert(t_hi == t_lo, "must be same");
 750     move_regs(f_lo, t_lo);
 751 
 752   } else if (dest->is_single_fpu()) {
 753     __ fmovs(dest->as_float_reg(), src->as_float_reg());
 754 
 755   } else if (dest->is_double_fpu()) {
 756     __ fmovd(dest->as_double_reg(), src->as_double_reg());
 757 
 758   } else {
 759     ShouldNotReachHere();
 760   }
 761 }
 762 
 763 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 764   precond(src->is_register() && dest->is_stack());
 765 
 766   uint const c_sz32 = sizeof(uint32_t);
 767   uint const c_sz64 = sizeof(uint64_t);
 768 
 769   if (src->is_single_cpu()) {
 770     int index = dest->single_stack_ix();
 771     if (is_reference_type(type)) {
 772       __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 773       __ verify_oop(src->as_register());
 774     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 775       __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 776     } else {
 777       __ strw(src->as_register(), stack_slot_address(index, c_sz32, rscratch1));
 778     }
 779 
 780   } else if (src->is_double_cpu()) {
 781     int index = dest->double_stack_ix();
 782     Address dest_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
 783     __ str(src->as_register_lo(), dest_addr_LO);
 784 
 785   } else if (src->is_single_fpu()) {
 786     int index = dest->single_stack_ix();
 787     __ strs(src->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
 788 
 789   } else if (src->is_double_fpu()) {
 790     int index = dest->double_stack_ix();
 791     __ strd(src->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
 792 
 793   } else {
 794     ShouldNotReachHere();
 795   }
 796 }
 797 
 798 
 799 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
 800   LIR_Address* to_addr = dest->as_address_ptr();
 801   PatchingStub* patch = NULL;
 802   Register compressed_src = rscratch1;
 803 
 804   if (patch_code != lir_patch_none) {
 805     deoptimize_trap(info);
 806     return;
 807   }
 808 
 809   if (is_reference_type(type)) {
 810     __ verify_oop(src->as_register());
 811 
 812     if (UseCompressedOops && !wide) {
 813       __ encode_heap_oop(compressed_src, src->as_register());
 814     } else {
 815       compressed_src = src->as_register();
 816     }
 817   }
 818 
 819   int null_check_here = code_offset();
 820   switch (type) {
 821     case T_FLOAT: {
 822       __ strs(src->as_float_reg(), as_Address(to_addr));
 823       break;
 824     }
 825 
 826     case T_DOUBLE: {
 827       __ strd(src->as_double_reg(), as_Address(to_addr));
 828       break;
 829     }
 830 
 831     case T_PRIMITIVE_OBJECT: // fall through
 832     case T_ARRAY:   // fall through
 833     case T_OBJECT:  // fall through
 834       if (UseCompressedOops && !wide) {
 835         __ strw(compressed_src, as_Address(to_addr, rscratch2));
 836       } else {
 837          __ str(compressed_src, as_Address(to_addr));
 838       }
 839       break;
 840     case T_METADATA:
 841       // We get here to store a method pointer to the stack to pass to
 842       // a dtrace runtime call. This can't work on 64 bit with
 843       // compressed klass ptrs: T_METADATA can be a compressed klass
 844       // ptr or a 64 bit method pointer.
 845       ShouldNotReachHere();
 846       __ str(src->as_register(), as_Address(to_addr));
 847       break;
 848     case T_ADDRESS:
 849       __ str(src->as_register(), as_Address(to_addr));
 850       break;
 851     case T_INT:
 852       __ strw(src->as_register(), as_Address(to_addr));
 853       break;
 854 
 855     case T_LONG: {
 856       __ str(src->as_register_lo(), as_Address_lo(to_addr));
 857       break;
 858     }
 859 
 860     case T_BYTE:    // fall through
 861     case T_BOOLEAN: {
 862       __ strb(src->as_register(), as_Address(to_addr));
 863       break;
 864     }
 865 
 866     case T_CHAR:    // fall through
 867     case T_SHORT:
 868       __ strh(src->as_register(), as_Address(to_addr));
 869       break;
 870 
 871     default:
 872       ShouldNotReachHere();
 873   }
 874   if (info != NULL) {
 875     add_debug_info_for_null_check(null_check_here, info);
 876   }
 877 }
 878 
 879 
 880 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 881   precond(src->is_stack() && dest->is_register());
 882 
 883   uint const c_sz32 = sizeof(uint32_t);
 884   uint const c_sz64 = sizeof(uint64_t);
 885 
 886   if (dest->is_single_cpu()) {
 887     int index = src->single_stack_ix();
 888     if (is_reference_type(type)) {
 889       __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 890       __ verify_oop(dest->as_register());
 891     } else if (type == T_METADATA || type == T_ADDRESS) {
 892       __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 893     } else {
 894       __ ldrw(dest->as_register(), stack_slot_address(index, c_sz32, rscratch1));
 895     }
 896 
 897   } else if (dest->is_double_cpu()) {
 898     int index = src->double_stack_ix();
 899     Address src_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
 900     __ ldr(dest->as_register_lo(), src_addr_LO);
 901 
 902   } else if (dest->is_single_fpu()) {
 903     int index = src->single_stack_ix();
 904     __ ldrs(dest->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
 905 
 906   } else if (dest->is_double_fpu()) {
 907     int index = src->double_stack_ix();
 908     __ ldrd(dest->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
 909 
 910   } else {
 911     ShouldNotReachHere();
 912   }
 913 }
 914 
 915 
 916 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 917   address target = NULL;
 918   relocInfo::relocType reloc_type = relocInfo::none;
 919 
 920   switch (patching_id(info)) {
 921   case PatchingStub::access_field_id:
 922     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 923     reloc_type = relocInfo::section_word_type;
 924     break;
 925   case PatchingStub::load_klass_id:
 926     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 927     reloc_type = relocInfo::metadata_type;
 928     break;
 929   case PatchingStub::load_mirror_id:
 930     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 931     reloc_type = relocInfo::oop_type;
 932     break;
 933   case PatchingStub::load_appendix_id:
 934     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 935     reloc_type = relocInfo::oop_type;
 936     break;
 937   default: ShouldNotReachHere();
 938   }
 939 
 940   __ far_call(RuntimeAddress(target));
 941   add_call_info_here(info);
 942 }
 943 
 944 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 945 
 946   LIR_Opr temp;
 947   if (type == T_LONG || type == T_DOUBLE)
 948     temp = FrameMap::rscratch1_long_opr;
 949   else
 950     temp = FrameMap::rscratch1_opr;
 951 
 952   stack2reg(src, temp, src->type());
 953   reg2stack(temp, dest, dest->type(), false);
 954 }
 955 
 956 
 957 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 958   LIR_Address* addr = src->as_address_ptr();
 959   LIR_Address* from_addr = src->as_address_ptr();
 960 
 961   if (addr->base()->type() == T_OBJECT || addr->base()->type() == T_PRIMITIVE_OBJECT) {
 962     __ verify_oop(addr->base()->as_pointer_register());
 963   }
 964 
 965   if (patch_code != lir_patch_none) {
 966     deoptimize_trap(info);
 967     return;
 968   }
 969 
 970   if (info != NULL) {
 971     add_debug_info_for_null_check_here(info);
 972   }
 973   int null_check_here = code_offset();
 974   switch (type) {
 975     case T_FLOAT: {
 976       __ ldrs(dest->as_float_reg(), as_Address(from_addr));
 977       break;
 978     }
 979 
 980     case T_DOUBLE: {
 981       __ ldrd(dest->as_double_reg(), as_Address(from_addr));
 982       break;
 983     }
 984 
 985     case T_PRIMITIVE_OBJECT: // fall through
 986     case T_ARRAY:   // fall through
 987     case T_OBJECT:  // fall through
 988       if (UseCompressedOops && !wide) {
 989         __ ldrw(dest->as_register(), as_Address(from_addr));
 990       } else {
 991         __ ldr(dest->as_register(), as_Address(from_addr));
 992       }
 993       break;
 994     case T_METADATA:
 995       // We get here to store a method pointer to the stack to pass to
 996       // a dtrace runtime call. This can't work on 64 bit with
 997       // compressed klass ptrs: T_METADATA can be a compressed klass
 998       // ptr or a 64 bit method pointer.
 999       ShouldNotReachHere();
1000       __ ldr(dest->as_register(), as_Address(from_addr));
1001       break;
1002     case T_ADDRESS:
1003       __ ldr(dest->as_register(), as_Address(from_addr));
1004       break;
1005     case T_INT:
1006       __ ldrw(dest->as_register(), as_Address(from_addr));
1007       break;
1008 
1009     case T_LONG: {
1010       __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1011       break;
1012     }
1013 
1014     case T_BYTE:
1015       __ ldrsb(dest->as_register(), as_Address(from_addr));
1016       break;
1017     case T_BOOLEAN: {
1018       __ ldrb(dest->as_register(), as_Address(from_addr));
1019       break;
1020     }
1021 
1022     case T_CHAR:
1023       __ ldrh(dest->as_register(), as_Address(from_addr));
1024       break;
1025     case T_SHORT:
1026       __ ldrsh(dest->as_register(), as_Address(from_addr));
1027       break;
1028 
1029     default:
1030       ShouldNotReachHere();
1031   }
1032 
1033   if (is_reference_type(type)) {
1034     if (UseCompressedOops && !wide) {
1035       __ decode_heap_oop(dest->as_register());
1036     }
1037 
1038     if (!UseZGC) {
1039       // Load barrier has not yet been applied, so ZGC can't verify the oop here
1040       __ verify_oop(dest->as_register());
1041     }
1042   }
1043 }
1044 
1045 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1046   assert(dst->is_cpu_register(), "must be");
1047   assert(dst->type() == src->type(), "must be");
1048 
1049   if (src->is_cpu_register()) {
1050     reg2reg(src, dst);
1051   } else if (src->is_stack()) {
1052     stack2reg(src, dst, dst->type());
1053   } else if (src->is_constant()) {
1054     const2reg(src, dst, lir_patch_none, NULL);
1055   } else {
1056     ShouldNotReachHere();
1057   }
1058 }
1059 
1060 int LIR_Assembler::array_element_size(BasicType type) const {
1061   int elem_size = type2aelembytes(type);
1062   return exact_log2(elem_size);
1063 }
1064 
1065 
1066 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1067   switch (op->code()) {
1068   case lir_idiv:
1069   case lir_irem:
1070     arithmetic_idiv(op->code(),
1071                     op->in_opr1(),
1072                     op->in_opr2(),
1073                     op->in_opr3(),
1074                     op->result_opr(),
1075                     op->info());
1076     break;
1077   case lir_fmad:
1078     __ fmaddd(op->result_opr()->as_double_reg(),
1079               op->in_opr1()->as_double_reg(),
1080               op->in_opr2()->as_double_reg(),
1081               op->in_opr3()->as_double_reg());
1082     break;
1083   case lir_fmaf:
1084     __ fmadds(op->result_opr()->as_float_reg(),
1085               op->in_opr1()->as_float_reg(),
1086               op->in_opr2()->as_float_reg(),
1087               op->in_opr3()->as_float_reg());
1088     break;
1089   default:      ShouldNotReachHere(); break;
1090   }
1091 }
1092 
1093 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1094 #ifdef ASSERT
1095   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1096   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1097   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1098 #endif
1099 
1100   if (op->cond() == lir_cond_always) {
1101     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1102     __ b(*(op->label()));
1103   } else {
1104     Assembler::Condition acond;
1105     if (op->code() == lir_cond_float_branch) {
1106       bool is_unordered = (op->ublock() == op->block());
1107       // Assembler::EQ does not permit unordered branches, so we add
1108       // another branch here.  Likewise, Assembler::NE does not permit
1109       // ordered branches.
1110       if ((is_unordered && op->cond() == lir_cond_equal)
1111           || (!is_unordered && op->cond() == lir_cond_notEqual))
1112         __ br(Assembler::VS, *(op->ublock()->label()));
1113       switch(op->cond()) {
1114       case lir_cond_equal:        acond = Assembler::EQ; break;
1115       case lir_cond_notEqual:     acond = Assembler::NE; break;
1116       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1117       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1118       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1119       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1120       default:                    ShouldNotReachHere();
1121         acond = Assembler::EQ;  // unreachable
1122       }
1123     } else {
1124       switch (op->cond()) {
1125         case lir_cond_equal:        acond = Assembler::EQ; break;
1126         case lir_cond_notEqual:     acond = Assembler::NE; break;
1127         case lir_cond_less:         acond = Assembler::LT; break;
1128         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1129         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1130         case lir_cond_greater:      acond = Assembler::GT; break;
1131         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1132         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1133         default:                    ShouldNotReachHere();
1134           acond = Assembler::EQ;  // unreachable
1135       }
1136     }
1137     __ br(acond,*(op->label()));
1138   }
1139 }
1140 
1141 
1142 
1143 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1144   LIR_Opr src  = op->in_opr();
1145   LIR_Opr dest = op->result_opr();
1146 
1147   switch (op->bytecode()) {
1148     case Bytecodes::_i2f:
1149       {
1150         __ scvtfws(dest->as_float_reg(), src->as_register());
1151         break;
1152       }
1153     case Bytecodes::_i2d:
1154       {
1155         __ scvtfwd(dest->as_double_reg(), src->as_register());
1156         break;
1157       }
1158     case Bytecodes::_l2d:
1159       {
1160         __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1161         break;
1162       }
1163     case Bytecodes::_l2f:
1164       {
1165         __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1166         break;
1167       }
1168     case Bytecodes::_f2d:
1169       {
1170         __ fcvts(dest->as_double_reg(), src->as_float_reg());
1171         break;
1172       }
1173     case Bytecodes::_d2f:
1174       {
1175         __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1176         break;
1177       }
1178     case Bytecodes::_i2c:
1179       {
1180         __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1181         break;
1182       }
1183     case Bytecodes::_i2l:
1184       {
1185         __ sxtw(dest->as_register_lo(), src->as_register());
1186         break;
1187       }
1188     case Bytecodes::_i2s:
1189       {
1190         __ sxth(dest->as_register(), src->as_register());
1191         break;
1192       }
1193     case Bytecodes::_i2b:
1194       {
1195         __ sxtb(dest->as_register(), src->as_register());
1196         break;
1197       }
1198     case Bytecodes::_l2i:
1199       {
1200         _masm->block_comment("FIXME: This could be a no-op");
1201         __ uxtw(dest->as_register(), src->as_register_lo());
1202         break;
1203       }
1204     case Bytecodes::_d2l:
1205       {
1206         __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1207         break;
1208       }
1209     case Bytecodes::_f2i:
1210       {
1211         __ fcvtzsw(dest->as_register(), src->as_float_reg());
1212         break;
1213       }
1214     case Bytecodes::_f2l:
1215       {
1216         __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1217         break;
1218       }
1219     case Bytecodes::_d2i:
1220       {
1221         __ fcvtzdw(dest->as_register(), src->as_double_reg());
1222         break;
1223       }
1224     default: ShouldNotReachHere();
1225   }
1226 }
1227 
1228 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1229   if (op->init_check()) {
1230     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1231                                InstanceKlass::init_state_offset()));
1232     __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1233     add_debug_info_for_null_check_here(op->stub()->info());
1234     __ br(Assembler::NE, *op->stub()->entry());
1235   }
1236   __ allocate_object(op->obj()->as_register(),
1237                      op->tmp1()->as_register(),
1238                      op->tmp2()->as_register(),
1239                      op->header_size(),
1240                      op->object_size(),
1241                      op->klass()->as_register(),
1242                      *op->stub()->entry());
1243   __ bind(*op->stub()->continuation());
1244 }
1245 
1246 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1247   Register len =  op->len()->as_register();
1248   __ uxtw(len, len);
1249 
1250   if (UseSlowPath || op->type() == T_PRIMITIVE_OBJECT ||
1251       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1252       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1253     __ b(*op->stub()->entry());
1254   } else {
1255     Register tmp1 = op->tmp1()->as_register();
1256     Register tmp2 = op->tmp2()->as_register();
1257     Register tmp3 = op->tmp3()->as_register();
1258     if (len == tmp1) {
1259       tmp1 = tmp3;
1260     } else if (len == tmp2) {
1261       tmp2 = tmp3;
1262     } else if (len == tmp3) {
1263       // everything is ok
1264     } else {
1265       __ mov(tmp3, len);
1266     }
1267     __ allocate_array(op->obj()->as_register(),
1268                       len,
1269                       tmp1,
1270                       tmp2,
1271                       arrayOopDesc::header_size(op->type()),
1272                       array_element_size(op->type()),
1273                       op->klass()->as_register(),
1274                       *op->stub()->entry());
1275   }
1276   __ bind(*op->stub()->continuation());
1277 }
1278 
1279 void LIR_Assembler::type_profile_helper(Register mdo,
1280                                         ciMethodData *md, ciProfileData *data,
1281                                         Register recv, Label* update_done) {
1282   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1283     Label next_test;
1284     // See if the receiver is receiver[n].
1285     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1286     __ ldr(rscratch1, Address(rscratch2));
1287     __ cmp(recv, rscratch1);
1288     __ br(Assembler::NE, next_test);
1289     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1290     __ addptr(data_addr, DataLayout::counter_increment);
1291     __ b(*update_done);
1292     __ bind(next_test);
1293   }
1294 
1295   // Didn't find receiver; find next empty slot and fill it in
1296   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1297     Label next_test;
1298     __ lea(rscratch2,
1299            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1300     Address recv_addr(rscratch2);
1301     __ ldr(rscratch1, recv_addr);
1302     __ cbnz(rscratch1, next_test);
1303     __ str(recv, recv_addr);
1304     __ mov(rscratch1, DataLayout::counter_increment);
1305     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1306     __ str(rscratch1, Address(rscratch2));
1307     __ b(*update_done);
1308     __ bind(next_test);
1309   }
1310 }
1311 
1312 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1313   // we always need a stub for the failure case.
1314   CodeStub* stub = op->stub();
1315   Register obj = op->object()->as_register();
1316   Register k_RInfo = op->tmp1()->as_register();
1317   Register klass_RInfo = op->tmp2()->as_register();
1318   Register dst = op->result_opr()->as_register();
1319   ciKlass* k = op->klass();
1320   Register Rtmp1 = noreg;
1321 
1322   // check if it needs to be profiled
1323   ciMethodData* md;
1324   ciProfileData* data;
1325 
1326   const bool should_profile = op->should_profile();
1327 
1328   if (should_profile) {
1329     ciMethod* method = op->profiled_method();
1330     assert(method != NULL, "Should have method");
1331     int bci = op->profiled_bci();
1332     md = method->method_data_or_null();
1333     assert(md != NULL, "Sanity");
1334     data = md->bci_to_data(bci);
1335     assert(data != NULL,                "need data for type check");
1336     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1337   }
1338   Label profile_cast_success, profile_cast_failure;
1339   Label *success_target = should_profile ? &profile_cast_success : success;
1340   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1341 
1342   if (obj == k_RInfo) {
1343     k_RInfo = dst;
1344   } else if (obj == klass_RInfo) {
1345     klass_RInfo = dst;
1346   }
1347   if (k->is_loaded() && !UseCompressedClassPointers) {
1348     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1349   } else {
1350     Rtmp1 = op->tmp3()->as_register();
1351     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1352   }
1353 
1354   assert_different_registers(obj, k_RInfo, klass_RInfo);
1355 
1356   if (op->need_null_check()) {
1357     if (should_profile) {
1358       Label not_null;
1359       __ cbnz(obj, not_null);
1360       // Object is null; update MDO and exit
1361       Register mdo  = klass_RInfo;
1362       __ mov_metadata(mdo, md->constant_encoding());
1363       Address data_addr
1364         = __ form_address(rscratch2, mdo,
1365                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1366                           0);
1367       __ ldrb(rscratch1, data_addr);
1368       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1369       __ strb(rscratch1, data_addr);
1370       __ b(*obj_is_null);
1371       __ bind(not_null);
1372     } else {
1373       __ cbz(obj, *obj_is_null);
1374     }
1375   }
1376 
1377   if (!k->is_loaded()) {
1378     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1379   } else {
1380     __ mov_metadata(k_RInfo, k->constant_encoding());
1381   }
1382   __ verify_oop(obj);
1383 
1384   if (op->fast_check()) {
1385     // get object class
1386     // not a safepoint as obj null check happens earlier
1387     __ load_klass(rscratch1, obj);
1388     __ cmp( rscratch1, k_RInfo);
1389 
1390     __ br(Assembler::NE, *failure_target);
1391     // successful cast, fall through to profile or jump
1392   } else {
1393     // get object class
1394     // not a safepoint as obj null check happens earlier
1395     __ load_klass(klass_RInfo, obj);
1396     if (k->is_loaded()) {
1397       // See if we get an immediate positive hit
1398       __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1399       __ cmp(k_RInfo, rscratch1);
1400       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1401         __ br(Assembler::NE, *failure_target);
1402         // successful cast, fall through to profile or jump
1403       } else {
1404         // See if we get an immediate positive hit
1405         __ br(Assembler::EQ, *success_target);
1406         // check for self
1407         __ cmp(klass_RInfo, k_RInfo);
1408         __ br(Assembler::EQ, *success_target);
1409 
1410         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1411         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1412         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1413         // result is a boolean
1414         __ cbzw(klass_RInfo, *failure_target);
1415         // successful cast, fall through to profile or jump
1416       }
1417     } else {
1418       // perform the fast part of the checking logic
1419       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1420       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1421       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1422       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1423       __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1424       // result is a boolean
1425       __ cbz(k_RInfo, *failure_target);
1426       // successful cast, fall through to profile or jump
1427     }
1428   }
1429   if (should_profile) {
1430     Register mdo  = klass_RInfo, recv = k_RInfo;
1431     __ bind(profile_cast_success);
1432     __ mov_metadata(mdo, md->constant_encoding());
1433     __ load_klass(recv, obj);
1434     Label update_done;
1435     type_profile_helper(mdo, md, data, recv, success);
1436     __ b(*success);
1437 
1438     __ bind(profile_cast_failure);
1439     __ mov_metadata(mdo, md->constant_encoding());
1440     Address counter_addr
1441       = __ form_address(rscratch2, mdo,
1442                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1443                         0);
1444     __ ldr(rscratch1, counter_addr);
1445     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1446     __ str(rscratch1, counter_addr);
1447     __ b(*failure);
1448   }
1449   __ b(*success);
1450 }
1451 
1452 
1453 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1454   const bool should_profile = op->should_profile();
1455 
1456   LIR_Code code = op->code();
1457   if (code == lir_store_check) {
1458     Register value = op->object()->as_register();
1459     Register array = op->array()->as_register();
1460     Register k_RInfo = op->tmp1()->as_register();
1461     Register klass_RInfo = op->tmp2()->as_register();
1462     Register Rtmp1 = op->tmp3()->as_register();
1463 
1464     CodeStub* stub = op->stub();
1465 
1466     // check if it needs to be profiled
1467     ciMethodData* md;
1468     ciProfileData* data;
1469 
1470     if (should_profile) {
1471       ciMethod* method = op->profiled_method();
1472       assert(method != NULL, "Should have method");
1473       int bci = op->profiled_bci();
1474       md = method->method_data_or_null();
1475       assert(md != NULL, "Sanity");
1476       data = md->bci_to_data(bci);
1477       assert(data != NULL,                "need data for type check");
1478       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1479     }
1480     Label profile_cast_success, profile_cast_failure, done;
1481     Label *success_target = should_profile ? &profile_cast_success : &done;
1482     Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1483 
1484     if (should_profile) {
1485       Label not_null;
1486       __ cbnz(value, not_null);
1487       // Object is null; update MDO and exit
1488       Register mdo  = klass_RInfo;
1489       __ mov_metadata(mdo, md->constant_encoding());
1490       Address data_addr
1491         = __ form_address(rscratch2, mdo,
1492                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1493                           0);
1494       __ ldrb(rscratch1, data_addr);
1495       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1496       __ strb(rscratch1, data_addr);
1497       __ b(done);
1498       __ bind(not_null);
1499     } else {
1500       __ cbz(value, done);
1501     }
1502 
1503     add_debug_info_for_null_check_here(op->info_for_exception());
1504     __ load_klass(k_RInfo, array);
1505     __ load_klass(klass_RInfo, value);
1506 
1507     // get instance klass (it's already uncompressed)
1508     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1509     // perform the fast part of the checking logic
1510     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1511     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1512     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1513     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1514     __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1515     // result is a boolean
1516     __ cbzw(k_RInfo, *failure_target);
1517     // fall through to the success case
1518 
1519     if (should_profile) {
1520       Register mdo  = klass_RInfo, recv = k_RInfo;
1521       __ bind(profile_cast_success);
1522       __ mov_metadata(mdo, md->constant_encoding());
1523       __ load_klass(recv, value);
1524       Label update_done;
1525       type_profile_helper(mdo, md, data, recv, &done);
1526       __ b(done);
1527 
1528       __ bind(profile_cast_failure);
1529       __ mov_metadata(mdo, md->constant_encoding());
1530       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1531       __ lea(rscratch2, counter_addr);
1532       __ ldr(rscratch1, Address(rscratch2));
1533       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1534       __ str(rscratch1, Address(rscratch2));
1535       __ b(*stub->entry());
1536     }
1537 
1538     __ bind(done);
1539   } else if (code == lir_checkcast) {
1540     Register obj = op->object()->as_register();
1541     Register dst = op->result_opr()->as_register();
1542     Label success;
1543     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1544     __ bind(success);
1545     if (dst != obj) {
1546       __ mov(dst, obj);
1547     }
1548   } else if (code == lir_instanceof) {
1549     Register obj = op->object()->as_register();
1550     Register dst = op->result_opr()->as_register();
1551     Label success, failure, done;
1552     emit_typecheck_helper(op, &success, &failure, &failure);
1553     __ bind(failure);
1554     __ mov(dst, zr);
1555     __ b(done);
1556     __ bind(success);
1557     __ mov(dst, 1);
1558     __ bind(done);
1559   } else {
1560     ShouldNotReachHere();
1561   }
1562 }
1563 
1564 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1565   // We are loading/storing from/to an array that *may* be flattened (the
1566   // declared type is Object[], abstract[], interface[] or VT.ref[]).
1567   // If this array is flattened, take the slow path.
1568 
1569   Register klass = op->tmp()->as_register();
1570   if (UseArrayMarkWordCheck) {
1571     __ test_flattened_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1572   } else {
1573     __ load_klass(klass, op->array()->as_register());
1574     __ ldrw(klass, Address(klass, Klass::layout_helper_offset()));
1575     __ tst(klass, Klass::_lh_array_tag_flat_value_bit_inplace);
1576     __ br(Assembler::NE, *op->stub()->entry());
1577   }
1578   if (!op->value()->is_illegal()) {
1579     // The array is not flattened, but it might be null-free. If we are storing
1580     // a null into a null-free array, take the slow path (which will throw NPE).
1581     Label skip;
1582     __ cbnz(op->value()->as_register(), skip);
1583     if (UseArrayMarkWordCheck) {
1584       __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1585     } else {
1586       __ tst(klass, Klass::_lh_null_free_array_bit_inplace);
1587       __ br(Assembler::NE, *op->stub()->entry());
1588     }
1589     __ bind(skip);
1590   }
1591 }
1592 
1593 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1594   // We are storing into an array that *may* be null-free (the declared type is
1595   // Object[], abstract[], interface[] or VT.ref[]).
1596   if (UseArrayMarkWordCheck) {
1597     Label test_mark_word;
1598     Register tmp = op->tmp()->as_register();
1599     __ ldr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1600     __ tst(tmp, markWord::unlocked_value);
1601     __ br(Assembler::NE, test_mark_word);
1602     __ load_prototype_header(tmp, op->array()->as_register());
1603     __ bind(test_mark_word);
1604     __ tst(tmp, markWord::null_free_array_bit_in_place);
1605   } else {
1606     Register klass = op->tmp()->as_register();
1607     __ load_klass(klass, op->array()->as_register());
1608     __ ldr(klass, Address(klass, Klass::layout_helper_offset()));
1609     __ tst(klass, Klass::_lh_null_free_array_bit_inplace);
1610   }
1611 }
1612 
1613 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1614   Label L_oops_equal;
1615   Label L_oops_not_equal;
1616   Label L_end;
1617 
1618   Register left  = op->left()->as_register();
1619   Register right = op->right()->as_register();
1620 
1621   __ cmp(left, right);
1622   __ br(Assembler::EQ, L_oops_equal);
1623 
1624   // (1) Null check -- if one of the operands is null, the other must not be null (because
1625   //     the two references are not equal), so they are not substitutable,
1626   //     FIXME: do null check only if the operand is nullable
1627   {
1628     __ cbz(left, L_oops_not_equal);
1629     __ cbz(right, L_oops_not_equal);
1630   }
1631 
1632   ciKlass* left_klass = op->left_klass();
1633   ciKlass* right_klass = op->right_klass();
1634 
1635   // (2) Inline type check -- if either of the operands is not a inline type,
1636   //     they are not substitutable. We do this only if we are not sure that the
1637   //     operands are inline type
1638   if ((left_klass == NULL || right_klass == NULL) ||// The klass is still unloaded, or came from a Phi node.
1639       !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1640     Register tmp1  = op->tmp1()->as_register();
1641     __ mov(tmp1, markWord::inline_type_pattern);
1642     __ ldr(rscratch1, Address(left, oopDesc::mark_offset_in_bytes()));
1643     __ andr(tmp1, tmp1, rscratch1);
1644     __ ldr(rscratch1, Address(right, oopDesc::mark_offset_in_bytes()));
1645     __ andr(tmp1, tmp1, rscratch1);
1646     __ cmp(tmp1, (u1)markWord::inline_type_pattern);
1647     __ br(Assembler::NE, L_oops_not_equal);
1648   }
1649 
1650   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1651   if (left_klass != NULL && left_klass->is_inlinetype() && left_klass == right_klass) {
1652     // No need to load klass -- the operands are statically known to be the same inline klass.
1653     __ b(*op->stub()->entry());
1654   } else {
1655     Register left_klass_op = op->left_klass_op()->as_register();
1656     Register right_klass_op = op->right_klass_op()->as_register();
1657 
1658     if (UseCompressedClassPointers) {
1659       __ ldrw(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1660       __ ldrw(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1661       __ cmpw(left_klass_op, right_klass_op);
1662     } else {
1663       __ ldr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1664       __ ldr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1665       __ cmp(left_klass_op, right_klass_op);
1666     }
1667 
1668     __ br(Assembler::EQ, *op->stub()->entry()); // same klass -> do slow check
1669     // fall through to L_oops_not_equal
1670   }
1671 
1672   __ bind(L_oops_not_equal);
1673   move(op->not_equal_result(), op->result_opr());
1674   __ b(L_end);
1675 
1676   __ bind(L_oops_equal);
1677   move(op->equal_result(), op->result_opr());
1678   __ b(L_end);
1679 
1680   // We've returned from the stub. R0 contains 0x0 IFF the two
1681   // operands are not substitutable. (Don't compare against 0x1 in case the
1682   // C compiler is naughty)
1683   __ bind(*op->stub()->continuation());
1684   __ cbz(r0, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1685   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1686   // fall-through
1687   __ bind(L_end);
1688 }
1689 
1690 
1691 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1692   __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1693   __ cset(rscratch1, Assembler::NE);
1694   __ membar(__ AnyAny);
1695 }
1696 
1697 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1698   __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1699   __ cset(rscratch1, Assembler::NE);
1700   __ membar(__ AnyAny);
1701 }
1702 
1703 
1704 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1705   assert(VM_Version::supports_cx8(), "wrong machine");
1706   Register addr;
1707   if (op->addr()->is_register()) {
1708     addr = as_reg(op->addr());
1709   } else {
1710     assert(op->addr()->is_address(), "what else?");
1711     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1712     assert(addr_ptr->disp() == 0, "need 0 disp");
1713     assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1714     addr = as_reg(addr_ptr->base());
1715   }
1716   Register newval = as_reg(op->new_value());
1717   Register cmpval = as_reg(op->cmp_value());
1718 
1719   if (op->code() == lir_cas_obj) {
1720     if (UseCompressedOops) {
1721       Register t1 = op->tmp1()->as_register();
1722       assert(op->tmp1()->is_valid(), "must be");
1723       __ encode_heap_oop(t1, cmpval);
1724       cmpval = t1;
1725       __ encode_heap_oop(rscratch2, newval);
1726       newval = rscratch2;
1727       casw(addr, newval, cmpval);
1728     } else {
1729       casl(addr, newval, cmpval);
1730     }
1731   } else if (op->code() == lir_cas_int) {
1732     casw(addr, newval, cmpval);
1733   } else {
1734     casl(addr, newval, cmpval);
1735   }
1736 }
1737 
1738 
1739 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1740                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1741   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on aarch64");
1742 
1743   Assembler::Condition acond, ncond;
1744   switch (condition) {
1745   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1746   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1747   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1748   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1749   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1750   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1751   case lir_cond_belowEqual:
1752   case lir_cond_aboveEqual:
1753   default:                    ShouldNotReachHere();
1754     acond = Assembler::EQ; ncond = Assembler::NE;  // unreachable
1755   }
1756 
1757   assert(result->is_single_cpu() || result->is_double_cpu(),
1758          "expect single register for result");
1759   if (opr1->is_constant() && opr2->is_constant()
1760       && opr1->type() == T_INT && opr2->type() == T_INT) {
1761     jint val1 = opr1->as_jint();
1762     jint val2 = opr2->as_jint();
1763     if (val1 == 0 && val2 == 1) {
1764       __ cset(result->as_register(), ncond);
1765       return;
1766     } else if (val1 == 1 && val2 == 0) {
1767       __ cset(result->as_register(), acond);
1768       return;
1769     }
1770   }
1771 
1772   if (opr1->is_constant() && opr2->is_constant()
1773       && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1774     jlong val1 = opr1->as_jlong();
1775     jlong val2 = opr2->as_jlong();
1776     if (val1 == 0 && val2 == 1) {
1777       __ cset(result->as_register_lo(), ncond);
1778       return;
1779     } else if (val1 == 1 && val2 == 0) {
1780       __ cset(result->as_register_lo(), acond);
1781       return;
1782     }
1783   }
1784 
1785   if (opr1->is_stack()) {
1786     stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1787     opr1 = FrameMap::rscratch1_opr;
1788   } else if (opr1->is_constant()) {
1789     LIR_Opr tmp
1790       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1791     const2reg(opr1, tmp, lir_patch_none, NULL);
1792     opr1 = tmp;
1793   }
1794 
1795   if (opr2->is_stack()) {
1796     stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1797     opr2 = FrameMap::rscratch2_opr;
1798   } else if (opr2->is_constant()) {
1799     LIR_Opr tmp
1800       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1801     const2reg(opr2, tmp, lir_patch_none, NULL);
1802     opr2 = tmp;
1803   }
1804 
1805   if (result->type() == T_LONG)
1806     __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1807   else
1808     __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1809 }
1810 
1811 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1812   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1813 
1814   if (left->is_single_cpu()) {
1815     Register lreg = left->as_register();
1816     Register dreg = as_reg(dest);
1817 
1818     if (right->is_single_cpu()) {
1819       // cpu register - cpu register
1820 
1821       assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1822              "should be");
1823       Register rreg = right->as_register();
1824       switch (code) {
1825       case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1826       case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1827       case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1828       default:      ShouldNotReachHere();
1829       }
1830 
1831     } else if (right->is_double_cpu()) {
1832       Register rreg = right->as_register_lo();
1833       // single_cpu + double_cpu: can happen with obj+long
1834       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1835       switch (code) {
1836       case lir_add: __ add(dreg, lreg, rreg); break;
1837       case lir_sub: __ sub(dreg, lreg, rreg); break;
1838       default: ShouldNotReachHere();
1839       }
1840     } else if (right->is_constant()) {
1841       // cpu register - constant
1842       jlong c;
1843 
1844       // FIXME.  This is fugly: we really need to factor all this logic.
1845       switch(right->type()) {
1846       case T_LONG:
1847         c = right->as_constant_ptr()->as_jlong();
1848         break;
1849       case T_INT:
1850       case T_ADDRESS:
1851         c = right->as_constant_ptr()->as_jint();
1852         break;
1853       default:
1854         ShouldNotReachHere();
1855         c = 0;  // unreachable
1856         break;
1857       }
1858 
1859       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1860       if (c == 0 && dreg == lreg) {
1861         COMMENT("effective nop elided");
1862         return;
1863       }
1864       switch(left->type()) {
1865       case T_INT:
1866         switch (code) {
1867         case lir_add: __ addw(dreg, lreg, c); break;
1868         case lir_sub: __ subw(dreg, lreg, c); break;
1869         default: ShouldNotReachHere();
1870         }
1871         break;
1872       case T_OBJECT:
1873       case T_ADDRESS:
1874         switch (code) {
1875         case lir_add: __ add(dreg, lreg, c); break;
1876         case lir_sub: __ sub(dreg, lreg, c); break;
1877         default: ShouldNotReachHere();
1878         }
1879         break;
1880       default:
1881         ShouldNotReachHere();
1882       }
1883     } else {
1884       ShouldNotReachHere();
1885     }
1886 
1887   } else if (left->is_double_cpu()) {
1888     Register lreg_lo = left->as_register_lo();
1889 
1890     if (right->is_double_cpu()) {
1891       // cpu register - cpu register
1892       Register rreg_lo = right->as_register_lo();
1893       switch (code) {
1894       case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1895       case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1896       case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1897       case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1898       case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1899       default:
1900         ShouldNotReachHere();
1901       }
1902 
1903     } else if (right->is_constant()) {
1904       jlong c = right->as_constant_ptr()->as_jlong();
1905       Register dreg = as_reg(dest);
1906       switch (code) {
1907         case lir_add:
1908         case lir_sub:
1909           if (c == 0 && dreg == lreg_lo) {
1910             COMMENT("effective nop elided");
1911             return;
1912           }
1913           code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c);
1914           break;
1915         case lir_div:
1916           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1917           if (c == 1) {
1918             // move lreg_lo to dreg if divisor is 1
1919             __ mov(dreg, lreg_lo);
1920           } else {
1921             unsigned int shift = log2i_exact(c);
1922             // use rscratch1 as intermediate result register
1923             __ asr(rscratch1, lreg_lo, 63);
1924             __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift);
1925             __ asr(dreg, rscratch1, shift);
1926           }
1927           break;
1928         case lir_rem:
1929           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1930           if (c == 1) {
1931             // move 0 to dreg if divisor is 1
1932             __ mov(dreg, zr);
1933           } else {
1934             // use rscratch1 as intermediate result register
1935             __ negs(rscratch1, lreg_lo);
1936             __ andr(dreg, lreg_lo, c - 1);
1937             __ andr(rscratch1, rscratch1, c - 1);
1938             __ csneg(dreg, dreg, rscratch1, Assembler::MI);
1939           }
1940           break;
1941         default:
1942           ShouldNotReachHere();
1943       }
1944     } else {
1945       ShouldNotReachHere();
1946     }
1947   } else if (left->is_single_fpu()) {
1948     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1949     switch (code) {
1950     case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1951     case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1952     case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1953     case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1954     default:
1955       ShouldNotReachHere();
1956     }
1957   } else if (left->is_double_fpu()) {
1958     if (right->is_double_fpu()) {
1959       // fpu register - fpu register
1960       switch (code) {
1961       case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1962       case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1963       case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1964       case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1965       default:
1966         ShouldNotReachHere();
1967       }
1968     } else {
1969       if (right->is_constant()) {
1970         ShouldNotReachHere();
1971       }
1972       ShouldNotReachHere();
1973     }
1974   } else if (left->is_single_stack() || left->is_address()) {
1975     assert(left == dest, "left and dest must be equal");
1976     ShouldNotReachHere();
1977   } else {
1978     ShouldNotReachHere();
1979   }
1980 }
1981 
1982 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1983 
1984 
1985 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1986   switch(code) {
1987   case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1988   case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1989   default      : ShouldNotReachHere();
1990   }
1991 }
1992 
1993 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1994 
1995   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1996   Register Rleft = left->is_single_cpu() ? left->as_register() :
1997                                            left->as_register_lo();
1998    if (dst->is_single_cpu()) {
1999      Register Rdst = dst->as_register();
2000      if (right->is_constant()) {
2001        switch (code) {
2002          case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
2003          case lir_logic_or:  __ orrw (Rdst, Rleft, right->as_jint()); break;
2004          case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
2005          default: ShouldNotReachHere(); break;
2006        }
2007      } else {
2008        Register Rright = right->is_single_cpu() ? right->as_register() :
2009                                                   right->as_register_lo();
2010        switch (code) {
2011          case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
2012          case lir_logic_or:  __ orrw (Rdst, Rleft, Rright); break;
2013          case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
2014          default: ShouldNotReachHere(); break;
2015        }
2016      }
2017    } else {
2018      Register Rdst = dst->as_register_lo();
2019      if (right->is_constant()) {
2020        switch (code) {
2021          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
2022          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jlong()); break;
2023          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
2024          default: ShouldNotReachHere(); break;
2025        }
2026      } else {
2027        Register Rright = right->is_single_cpu() ? right->as_register() :
2028                                                   right->as_register_lo();
2029        switch (code) {
2030          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
2031          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
2032          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
2033          default: ShouldNotReachHere(); break;
2034        }
2035      }
2036    }
2037 }
2038 
2039 
2040 
2041 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) {
2042 
2043   // opcode check
2044   assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem");
2045   bool is_irem = (code == lir_irem);
2046 
2047   // operand check
2048   assert(left->is_single_cpu(),   "left must be register");
2049   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
2050   assert(result->is_single_cpu(), "result must be register");
2051   Register lreg = left->as_register();
2052   Register dreg = result->as_register();
2053 
2054   // power-of-2 constant check and codegen
2055   if (right->is_constant()) {
2056     int c = right->as_constant_ptr()->as_jint();
2057     assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
2058     if (is_irem) {
2059       if (c == 1) {
2060         // move 0 to dreg if divisor is 1
2061         __ movw(dreg, zr);
2062       } else {
2063         // use rscratch1 as intermediate result register
2064         __ negsw(rscratch1, lreg);
2065         __ andw(dreg, lreg, c - 1);
2066         __ andw(rscratch1, rscratch1, c - 1);
2067         __ csnegw(dreg, dreg, rscratch1, Assembler::MI);
2068       }
2069     } else {
2070       if (c == 1) {
2071         // move lreg to dreg if divisor is 1
2072         __ movw(dreg, lreg);
2073       } else {
2074         unsigned int shift = exact_log2(c);
2075         // use rscratch1 as intermediate result register
2076         __ asrw(rscratch1, lreg, 31);
2077         __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift);
2078         __ asrw(dreg, rscratch1, shift);
2079       }
2080     }
2081   } else {
2082     Register rreg = right->as_register();
2083     __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1);
2084   }
2085 }
2086 
2087 
2088 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
2089   if (opr1->is_constant() && opr2->is_single_cpu()) {
2090     // tableswitch
2091     Register reg = as_reg(opr2);
2092     struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
2093     __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
2094   } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
2095     Register reg1 = as_reg(opr1);
2096     if (opr2->is_single_cpu()) {
2097       // cpu register - cpu register
2098       Register reg2 = opr2->as_register();
2099       if (is_reference_type(opr1->type())) {
2100         __ cmpoop(reg1, reg2);
2101       } else {
2102         assert(!is_reference_type(opr2->type()), "cmp int, oop?");
2103         __ cmpw(reg1, reg2);
2104       }
2105       return;
2106     }
2107     if (opr2->is_double_cpu()) {
2108       // cpu register - cpu register
2109       Register reg2 = opr2->as_register_lo();
2110       __ cmp(reg1, reg2);
2111       return;
2112     }
2113 
2114     if (opr2->is_constant()) {
2115       bool is_32bit = false; // width of register operand
2116       jlong imm;
2117 
2118       switch(opr2->type()) {
2119       case T_INT:
2120         imm = opr2->as_constant_ptr()->as_jint();
2121         is_32bit = true;
2122         break;
2123       case T_LONG:
2124         imm = opr2->as_constant_ptr()->as_jlong();
2125         break;
2126       case T_ADDRESS:
2127         imm = opr2->as_constant_ptr()->as_jint();
2128         break;
2129       case T_METADATA:
2130         imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
2131         break;
2132       case T_PRIMITIVE_OBJECT:
2133       case T_OBJECT:
2134       case T_ARRAY:
2135         jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
2136         __ cmpoop(reg1, rscratch1);
2137         return;
2138       default:
2139         ShouldNotReachHere();
2140         imm = 0;  // unreachable
2141         break;
2142       }
2143 
2144       if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
2145         if (is_32bit)
2146           __ cmpw(reg1, imm);
2147         else
2148           __ subs(zr, reg1, imm);
2149         return;
2150       } else {
2151         __ mov(rscratch1, imm);
2152         if (is_32bit)
2153           __ cmpw(reg1, rscratch1);
2154         else
2155           __ cmp(reg1, rscratch1);
2156         return;
2157       }
2158     } else
2159       ShouldNotReachHere();
2160   } else if (opr1->is_single_fpu()) {
2161     FloatRegister reg1 = opr1->as_float_reg();
2162     assert(opr2->is_single_fpu(), "expect single float register");
2163     FloatRegister reg2 = opr2->as_float_reg();
2164     __ fcmps(reg1, reg2);
2165   } else if (opr1->is_double_fpu()) {
2166     FloatRegister reg1 = opr1->as_double_reg();
2167     assert(opr2->is_double_fpu(), "expect double float register");
2168     FloatRegister reg2 = opr2->as_double_reg();
2169     __ fcmpd(reg1, reg2);
2170   } else {
2171     ShouldNotReachHere();
2172   }
2173 }
2174 
2175 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2176   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2177     bool is_unordered_less = (code == lir_ucmp_fd2i);
2178     if (left->is_single_fpu()) {
2179       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2180     } else if (left->is_double_fpu()) {
2181       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2182     } else {
2183       ShouldNotReachHere();
2184     }
2185   } else if (code == lir_cmp_l2i) {
2186     Label done;
2187     __ cmp(left->as_register_lo(), right->as_register_lo());
2188     __ mov(dst->as_register(), (uint64_t)-1L);
2189     __ br(Assembler::LT, done);
2190     __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2191     __ bind(done);
2192   } else {
2193     ShouldNotReachHere();
2194   }
2195 }
2196 
2197 
2198 void LIR_Assembler::align_call(LIR_Code code) {  }
2199 
2200 
2201 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2202   address call = __ trampoline_call(Address(op->addr(), rtype));
2203   if (call == NULL) {
2204     bailout("trampoline stub overflow");
2205     return;
2206   }
2207   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2208   __ post_call_nop();
2209 }
2210 
2211 
2212 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2213   address call = __ ic_call(op->addr());
2214   if (call == NULL) {
2215     bailout("trampoline stub overflow");
2216     return;
2217   }
2218   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2219   __ post_call_nop();
2220 }
2221 
2222 void LIR_Assembler::emit_static_call_stub() {
2223   address call_pc = __ pc();
2224   address stub = __ start_a_stub(call_stub_size());
2225   if (stub == NULL) {
2226     bailout("static call stub overflow");
2227     return;
2228   }
2229 
2230   int start = __ offset();
2231 
2232   __ relocate(static_stub_Relocation::spec(call_pc));
2233   __ emit_static_call_stub();
2234 
2235   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
2236         <= call_stub_size(), "stub too big");
2237   __ end_a_stub();
2238 }
2239 
2240 
2241 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2242   assert(exceptionOop->as_register() == r0, "must match");
2243   assert(exceptionPC->as_register() == r3, "must match");
2244 
2245   // exception object is not added to oop map by LinearScan
2246   // (LinearScan assumes that no oops are in fixed registers)
2247   info->add_register_oop(exceptionOop);
2248   Runtime1::StubID unwind_id;
2249 
2250   // get current pc information
2251   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2252   if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
2253     // As no instructions have been generated yet for this LIR node it's
2254     // possible that an oop map already exists for the current offset.
2255     // In that case insert an dummy NOP here to ensure all oop map PCs
2256     // are unique. See JDK-8237483.
2257     __ nop();
2258   }
2259   int pc_for_athrow_offset = __ offset();
2260   InternalAddress pc_for_athrow(__ pc());
2261   __ adr(exceptionPC->as_register(), pc_for_athrow);
2262   add_call_info(pc_for_athrow_offset, info); // for exception handler
2263 
2264   __ verify_not_null_oop(r0);
2265   // search an exception handler (r0: exception oop, r3: throwing pc)
2266   if (compilation()->has_fpu_code()) {
2267     unwind_id = Runtime1::handle_exception_id;
2268   } else {
2269     unwind_id = Runtime1::handle_exception_nofpu_id;
2270   }
2271   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2272 
2273   // FIXME: enough room for two byte trap   ????
2274   __ nop();
2275 }
2276 
2277 
2278 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2279   assert(exceptionOop->as_register() == r0, "must match");
2280 
2281   __ b(_unwind_handler_entry);
2282 }
2283 
2284 
2285 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2286   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2287   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2288 
2289   switch (left->type()) {
2290     case T_INT: {
2291       switch (code) {
2292       case lir_shl:  __ lslvw (dreg, lreg, count->as_register()); break;
2293       case lir_shr:  __ asrvw (dreg, lreg, count->as_register()); break;
2294       case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2295       default:
2296         ShouldNotReachHere();
2297         break;
2298       }
2299       break;
2300     case T_LONG:
2301     case T_PRIMITIVE_OBJECT:
2302     case T_ADDRESS:
2303     case T_OBJECT:
2304       switch (code) {
2305       case lir_shl:  __ lslv (dreg, lreg, count->as_register()); break;
2306       case lir_shr:  __ asrv (dreg, lreg, count->as_register()); break;
2307       case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2308       default:
2309         ShouldNotReachHere();
2310         break;
2311       }
2312       break;
2313     default:
2314       ShouldNotReachHere();
2315       break;
2316     }
2317   }
2318 }
2319 
2320 
2321 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2322   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2323   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2324 
2325   switch (left->type()) {
2326     case T_INT: {
2327       switch (code) {
2328       case lir_shl:  __ lslw (dreg, lreg, count); break;
2329       case lir_shr:  __ asrw (dreg, lreg, count); break;
2330       case lir_ushr: __ lsrw (dreg, lreg, count); break;
2331       default:
2332         ShouldNotReachHere();
2333         break;
2334       }
2335       break;
2336     case T_LONG:
2337     case T_ADDRESS:
2338     case T_PRIMITIVE_OBJECT:
2339     case T_OBJECT:
2340       switch (code) {
2341       case lir_shl:  __ lsl (dreg, lreg, count); break;
2342       case lir_shr:  __ asr (dreg, lreg, count); break;
2343       case lir_ushr: __ lsr (dreg, lreg, count); break;
2344       default:
2345         ShouldNotReachHere();
2346         break;
2347       }
2348       break;
2349     default:
2350       ShouldNotReachHere();
2351       break;
2352     }
2353   }
2354 }
2355 
2356 
2357 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2358   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2359   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2360   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2361   __ str (r, Address(sp, offset_from_rsp_in_bytes));
2362 }
2363 
2364 
2365 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2366   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2367   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2368   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2369   __ mov (rscratch1, c);
2370   __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2371 }
2372 
2373 
2374 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2375   ShouldNotReachHere();
2376   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2377   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2378   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2379   __ lea(rscratch1, __ constant_oop_address(o));
2380   __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2381 }
2382 
2383 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2384   if (null_check) {
2385     __ cbz(obj, *slow_path->entry());
2386   }
2387   if (UseArrayMarkWordCheck) {
2388     if (is_dest) {
2389       __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2390     } else {
2391       __ test_flattened_array_oop(obj, tmp, *slow_path->entry());
2392     }
2393   } else {
2394     __ load_klass(tmp, obj);
2395     __ ldr(tmp, Address(tmp, Klass::layout_helper_offset()));
2396     if (is_dest) {
2397       // Take the slow path if it's a null_free destination array, in case the source array contains NULLs.
2398       __ tst(tmp, Klass::_lh_null_free_array_bit_inplace);
2399     } else {
2400       __ tst(tmp, Klass::_lh_array_tag_flat_value_bit_inplace);
2401     }
2402     __ br(Assembler::NE, *slow_path->entry());
2403   }
2404 }
2405 
2406 // This code replaces a call to arraycopy; no exception may
2407 // be thrown in this code, they must be thrown in the System.arraycopy
2408 // activation frame; we could save some checks if this would not be the case
2409 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2410   ciArrayKlass* default_type = op->expected_type();
2411   Register src = op->src()->as_register();
2412   Register dst = op->dst()->as_register();
2413   Register src_pos = op->src_pos()->as_register();
2414   Register dst_pos = op->dst_pos()->as_register();
2415   Register length  = op->length()->as_register();
2416   Register tmp = op->tmp()->as_register();
2417 
2418   CodeStub* stub = op->stub();
2419   int flags = op->flags();
2420   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2421   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2422 
2423   if (flags & LIR_OpArrayCopy::always_slow_path) {
2424     __ b(*stub->entry());
2425     __ bind(*stub->continuation());
2426     return;
2427   }
2428 
2429   // if we don't know anything, just go through the generic arraycopy
2430   if (default_type == NULL // || basic_type == T_OBJECT
2431       ) {
2432     Label done;
2433     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2434 
2435     // Save the arguments in case the generic arraycopy fails and we
2436     // have to fall back to the JNI stub
2437     __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2438     __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2439     __ str(src,              Address(sp, 4*BytesPerWord));
2440 
2441     address copyfunc_addr = StubRoutines::generic_arraycopy();
2442     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2443 
2444     // The arguments are in java calling convention so we shift them
2445     // to C convention
2446     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2447     __ mov(c_rarg0, j_rarg0);
2448     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2449     __ mov(c_rarg1, j_rarg1);
2450     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2451     __ mov(c_rarg2, j_rarg2);
2452     assert_different_registers(c_rarg3, j_rarg4);
2453     __ mov(c_rarg3, j_rarg3);
2454     __ mov(c_rarg4, j_rarg4);
2455 #ifndef PRODUCT
2456     if (PrintC1Statistics) {
2457       __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2458     }
2459 #endif
2460     __ far_call(RuntimeAddress(copyfunc_addr));
2461 
2462     __ cbz(r0, *stub->continuation());
2463 
2464     // Reload values from the stack so they are where the stub
2465     // expects them.
2466     __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2467     __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2468     __ ldr(src,              Address(sp, 4*BytesPerWord));
2469 
2470     // r0 is -1^K where K == partial copied count
2471     __ eonw(rscratch1, r0, zr);
2472     // adjust length down and src/end pos up by partial copied count
2473     __ subw(length, length, rscratch1);
2474     __ addw(src_pos, src_pos, rscratch1);
2475     __ addw(dst_pos, dst_pos, rscratch1);
2476     __ b(*stub->entry());
2477 
2478     __ bind(*stub->continuation());
2479     return;
2480   }
2481 
2482   // Handle inline type arrays
2483   if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2484     arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2485   }
2486 
2487   if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2488     arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2489   }
2490 
2491   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2492 
2493   int elem_size = type2aelembytes(basic_type);
2494   int scale = exact_log2(elem_size);
2495 
2496   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2497   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2498   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2499   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2500 
2501   // test for NULL
2502   if (flags & LIR_OpArrayCopy::src_null_check) {
2503     __ cbz(src, *stub->entry());
2504   }
2505   if (flags & LIR_OpArrayCopy::dst_null_check) {
2506     __ cbz(dst, *stub->entry());
2507   }
2508 
2509   // If the compiler was not able to prove that exact type of the source or the destination
2510   // of the arraycopy is an array type, check at runtime if the source or the destination is
2511   // an instance type.
2512   if (flags & LIR_OpArrayCopy::type_check) {
2513     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2514       __ load_klass(tmp, dst);
2515       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2516       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2517       __ br(Assembler::GE, *stub->entry());
2518     }
2519 
2520     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2521       __ load_klass(tmp, src);
2522       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2523       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2524       __ br(Assembler::GE, *stub->entry());
2525     }
2526   }
2527 
2528   // check if negative
2529   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2530     __ cmpw(src_pos, 0);
2531     __ br(Assembler::LT, *stub->entry());
2532   }
2533   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2534     __ cmpw(dst_pos, 0);
2535     __ br(Assembler::LT, *stub->entry());
2536   }
2537 
2538   if (flags & LIR_OpArrayCopy::length_positive_check) {
2539     __ cmpw(length, 0);
2540     __ br(Assembler::LT, *stub->entry());
2541   }
2542 
2543   if (flags & LIR_OpArrayCopy::src_range_check) {
2544     __ addw(tmp, src_pos, length);
2545     __ ldrw(rscratch1, src_length_addr);
2546     __ cmpw(tmp, rscratch1);
2547     __ br(Assembler::HI, *stub->entry());
2548   }
2549   if (flags & LIR_OpArrayCopy::dst_range_check) {
2550     __ addw(tmp, dst_pos, length);
2551     __ ldrw(rscratch1, dst_length_addr);
2552     __ cmpw(tmp, rscratch1);
2553     __ br(Assembler::HI, *stub->entry());
2554   }
2555 
2556   if (flags & LIR_OpArrayCopy::type_check) {
2557     // We don't know the array types are compatible
2558     if (basic_type != T_OBJECT) {
2559       // Simple test for basic type arrays
2560       if (UseCompressedClassPointers) {
2561         __ ldrw(tmp, src_klass_addr);
2562         __ ldrw(rscratch1, dst_klass_addr);
2563         __ cmpw(tmp, rscratch1);
2564       } else {
2565         __ ldr(tmp, src_klass_addr);
2566         __ ldr(rscratch1, dst_klass_addr);
2567         __ cmp(tmp, rscratch1);
2568       }
2569       __ br(Assembler::NE, *stub->entry());
2570     } else {
2571       // For object arrays, if src is a sub class of dst then we can
2572       // safely do the copy.
2573       Label cont, slow;
2574 
2575 #define PUSH(r1, r2)                                    \
2576       stp(r1, r2, __ pre(sp, -2 * wordSize));
2577 
2578 #define POP(r1, r2)                                     \
2579       ldp(r1, r2, __ post(sp, 2 * wordSize));
2580 
2581       __ PUSH(src, dst);
2582 
2583       __ load_klass(src, src);
2584       __ load_klass(dst, dst);
2585 
2586       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2587 
2588       __ PUSH(src, dst);
2589       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2590       __ POP(src, dst);
2591 
2592       __ cbnz(src, cont);
2593 
2594       __ bind(slow);
2595       __ POP(src, dst);
2596 
2597       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2598       if (copyfunc_addr != NULL) { // use stub if available
2599         // src is not a sub class of dst so we have to do a
2600         // per-element check.
2601 
2602         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2603         if ((flags & mask) != mask) {
2604           // Check that at least both of them object arrays.
2605           assert(flags & mask, "one of the two should be known to be an object array");
2606 
2607           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2608             __ load_klass(tmp, src);
2609           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2610             __ load_klass(tmp, dst);
2611           }
2612           int lh_offset = in_bytes(Klass::layout_helper_offset());
2613           Address klass_lh_addr(tmp, lh_offset);
2614           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2615           __ ldrw(rscratch1, klass_lh_addr);
2616           __ mov(rscratch2, objArray_lh);
2617           __ eorw(rscratch1, rscratch1, rscratch2);
2618           __ cbnzw(rscratch1, *stub->entry());
2619         }
2620 
2621        // Spill because stubs can use any register they like and it's
2622        // easier to restore just those that we care about.
2623         __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2624         __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2625         __ str(src,              Address(sp, 4*BytesPerWord));
2626 
2627         __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2628         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2629         assert_different_registers(c_rarg0, dst, dst_pos, length);
2630         __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2631         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2632         assert_different_registers(c_rarg1, dst, length);
2633         __ uxtw(c_rarg2, length);
2634         assert_different_registers(c_rarg2, dst);
2635 
2636         __ load_klass(c_rarg4, dst);
2637         __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2638         __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2639         __ far_call(RuntimeAddress(copyfunc_addr));
2640 
2641 #ifndef PRODUCT
2642         if (PrintC1Statistics) {
2643           Label failed;
2644           __ cbnz(r0, failed);
2645           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2646           __ bind(failed);
2647         }
2648 #endif
2649 
2650         __ cbz(r0, *stub->continuation());
2651 
2652 #ifndef PRODUCT
2653         if (PrintC1Statistics) {
2654           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2655         }
2656 #endif
2657         assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2658 
2659         // Restore previously spilled arguments
2660         __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2661         __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2662         __ ldr(src,              Address(sp, 4*BytesPerWord));
2663 
2664         // return value is -1^K where K is partial copied count
2665         __ eonw(rscratch1, r0, zr);
2666         // adjust length down and src/end pos up by partial copied count
2667         __ subw(length, length, rscratch1);
2668         __ addw(src_pos, src_pos, rscratch1);
2669         __ addw(dst_pos, dst_pos, rscratch1);
2670       }
2671 
2672       __ b(*stub->entry());
2673 
2674       __ bind(cont);
2675       __ POP(src, dst);
2676     }
2677   }
2678 
2679 #ifdef ASSERT
2680   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2681     // Sanity check the known type with the incoming class.  For the
2682     // primitive case the types must match exactly with src.klass and
2683     // dst.klass each exactly matching the default type.  For the
2684     // object array case, if no type check is needed then either the
2685     // dst type is exactly the expected type and the src type is a
2686     // subtype which we can't check or src is the same array as dst
2687     // but not necessarily exactly of type default_type.
2688     Label known_ok, halt;
2689     __ mov_metadata(tmp, default_type->constant_encoding());
2690     if (UseCompressedClassPointers) {
2691       __ encode_klass_not_null(tmp);
2692     }
2693 
2694     if (basic_type != T_OBJECT) {
2695 
2696       if (UseCompressedClassPointers) {
2697         __ ldrw(rscratch1, dst_klass_addr);
2698         __ cmpw(tmp, rscratch1);
2699       } else {
2700         __ ldr(rscratch1, dst_klass_addr);
2701         __ cmp(tmp, rscratch1);
2702       }
2703       __ br(Assembler::NE, halt);
2704       if (UseCompressedClassPointers) {
2705         __ ldrw(rscratch1, src_klass_addr);
2706         __ cmpw(tmp, rscratch1);
2707       } else {
2708         __ ldr(rscratch1, src_klass_addr);
2709         __ cmp(tmp, rscratch1);
2710       }
2711       __ br(Assembler::EQ, known_ok);
2712     } else {
2713       if (UseCompressedClassPointers) {
2714         __ ldrw(rscratch1, dst_klass_addr);
2715         __ cmpw(tmp, rscratch1);
2716       } else {
2717         __ ldr(rscratch1, dst_klass_addr);
2718         __ cmp(tmp, rscratch1);
2719       }
2720       __ br(Assembler::EQ, known_ok);
2721       __ cmp(src, dst);
2722       __ br(Assembler::EQ, known_ok);
2723     }
2724     __ bind(halt);
2725     __ stop("incorrect type information in arraycopy");
2726     __ bind(known_ok);
2727   }
2728 #endif
2729 
2730 #ifndef PRODUCT
2731   if (PrintC1Statistics) {
2732     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2733   }
2734 #endif
2735 
2736   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2737   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2738   assert_different_registers(c_rarg0, dst, dst_pos, length);
2739   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2740   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2741   assert_different_registers(c_rarg1, dst, length);
2742   __ uxtw(c_rarg2, length);
2743   assert_different_registers(c_rarg2, dst);
2744 
2745   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2746   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2747   const char *name;
2748   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2749 
2750  CodeBlob *cb = CodeCache::find_blob(entry);
2751  if (cb) {
2752    __ far_call(RuntimeAddress(entry));
2753  } else {
2754    __ call_VM_leaf(entry, 3);
2755  }
2756 
2757   __ bind(*stub->continuation());
2758 }
2759 
2760 
2761 
2762 
2763 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2764   Register obj = op->obj_opr()->as_register();  // may not be an oop
2765   Register hdr = op->hdr_opr()->as_register();
2766   Register lock = op->lock_opr()->as_register();
2767   if (UseHeavyMonitors) {
2768     if (op->info() != NULL) {
2769       add_debug_info_for_null_check_here(op->info());
2770       __ null_check(obj, -1);
2771     }
2772     __ b(*op->stub()->entry());
2773   } else if (op->code() == lir_lock) {
2774     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2775     // add debug info for NullPointerException only if one is possible
2776     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
2777     if (op->info() != NULL) {
2778       add_debug_info_for_null_check(null_check_offset, op->info());
2779     }
2780     // done
2781   } else if (op->code() == lir_unlock) {
2782     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2783     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2784   } else {
2785     Unimplemented();
2786   }
2787   __ bind(*op->stub()->continuation());
2788 }
2789 
2790 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2791   Register obj = op->obj()->as_pointer_register();
2792   Register result = op->result_opr()->as_pointer_register();
2793 
2794   CodeEmitInfo* info = op->info();
2795   if (info != NULL) {
2796     add_debug_info_for_null_check_here(info);
2797   }
2798 
2799   if (UseCompressedClassPointers) {
2800     __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2801     __ decode_klass_not_null(result);
2802   } else {
2803     __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2804   }
2805 }
2806 
2807 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2808   ciMethod* method = op->profiled_method();
2809   int bci          = op->profiled_bci();
2810   ciMethod* callee = op->profiled_callee();
2811 
2812   // Update counter for all call types
2813   ciMethodData* md = method->method_data_or_null();
2814   assert(md != NULL, "Sanity");
2815   ciProfileData* data = md->bci_to_data(bci);
2816   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2817   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2818   Register mdo  = op->mdo()->as_register();
2819   __ mov_metadata(mdo, md->constant_encoding());
2820   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2821   // Perform additional virtual call profiling for invokevirtual and
2822   // invokeinterface bytecodes
2823   if (op->should_profile_receiver_type()) {
2824     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2825     Register recv = op->recv()->as_register();
2826     assert_different_registers(mdo, recv);
2827     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2828     ciKlass* known_klass = op->known_holder();
2829     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2830       // We know the type that will be seen at this call site; we can
2831       // statically update the MethodData* rather than needing to do
2832       // dynamic tests on the receiver type
2833 
2834       // NOTE: we should probably put a lock around this search to
2835       // avoid collisions by concurrent compilations
2836       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2837       uint i;
2838       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2839         ciKlass* receiver = vc_data->receiver(i);
2840         if (known_klass->equals(receiver)) {
2841           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2842           __ addptr(data_addr, DataLayout::counter_increment);
2843           return;
2844         }
2845       }
2846 
2847       // Receiver type not found in profile data; select an empty slot
2848 
2849       // Note that this is less efficient than it should be because it
2850       // always does a write to the receiver part of the
2851       // VirtualCallData rather than just the first time
2852       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2853         ciKlass* receiver = vc_data->receiver(i);
2854         if (receiver == NULL) {
2855           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2856           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2857           __ lea(rscratch2, recv_addr);
2858           __ str(rscratch1, Address(rscratch2));
2859           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2860           __ addptr(data_addr, DataLayout::counter_increment);
2861           return;
2862         }
2863       }
2864     } else {
2865       __ load_klass(recv, recv);
2866       Label update_done;
2867       type_profile_helper(mdo, md, data, recv, &update_done);
2868       // Receiver did not match any saved receiver and there is no empty row for it.
2869       // Increment total counter to indicate polymorphic case.
2870       __ addptr(counter_addr, DataLayout::counter_increment);
2871 
2872       __ bind(update_done);
2873     }
2874   } else {
2875     // Static call
2876     __ addptr(counter_addr, DataLayout::counter_increment);
2877   }
2878 }
2879 
2880 
2881 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2882   Unimplemented();
2883 }
2884 
2885 
2886 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2887   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2888 }
2889 
2890 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2891   assert(op->crc()->is_single_cpu(),  "crc must be register");
2892   assert(op->val()->is_single_cpu(),  "byte value must be register");
2893   assert(op->result_opr()->is_single_cpu(), "result must be register");
2894   Register crc = op->crc()->as_register();
2895   Register val = op->val()->as_register();
2896   Register res = op->result_opr()->as_register();
2897 
2898   assert_different_registers(val, crc, res);
2899   uint64_t offset;
2900   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2901   __ add(res, res, offset);
2902 
2903   __ mvnw(crc, crc); // ~crc
2904   __ update_byte_crc32(crc, val, res);
2905   __ mvnw(res, crc); // ~crc
2906 }
2907 
2908 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2909   COMMENT("emit_profile_type {");
2910   Register obj = op->obj()->as_register();
2911   Register tmp = op->tmp()->as_pointer_register();
2912   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2913   ciKlass* exact_klass = op->exact_klass();
2914   intptr_t current_klass = op->current_klass();
2915   bool not_null = op->not_null();
2916   bool no_conflict = op->no_conflict();
2917 
2918   Label update, next, none;
2919 
2920   bool do_null = !not_null;
2921   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2922   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2923 
2924   assert(do_null || do_update, "why are we here?");
2925   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2926   assert(mdo_addr.base() != rscratch1, "wrong register");
2927 
2928   __ verify_oop(obj);
2929 
2930   if (tmp != obj) {
2931     __ mov(tmp, obj);
2932   }
2933   if (do_null) {
2934     __ cbnz(tmp, update);
2935     if (!TypeEntries::was_null_seen(current_klass)) {
2936       __ ldr(rscratch2, mdo_addr);
2937       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2938       __ str(rscratch2, mdo_addr);
2939     }
2940     if (do_update) {
2941 #ifndef ASSERT
2942       __ b(next);
2943     }
2944 #else
2945       __ b(next);
2946     }
2947   } else {
2948     __ cbnz(tmp, update);
2949     __ stop("unexpected null obj");
2950 #endif
2951   }
2952 
2953   __ bind(update);
2954 
2955   if (do_update) {
2956 #ifdef ASSERT
2957     if (exact_klass != NULL) {
2958       Label ok;
2959       __ load_klass(tmp, tmp);
2960       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2961       __ eor(rscratch1, tmp, rscratch1);
2962       __ cbz(rscratch1, ok);
2963       __ stop("exact klass and actual klass differ");
2964       __ bind(ok);
2965     }
2966 #endif
2967     if (!no_conflict) {
2968       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2969         if (exact_klass != NULL) {
2970           __ mov_metadata(tmp, exact_klass->constant_encoding());
2971         } else {
2972           __ load_klass(tmp, tmp);
2973         }
2974 
2975         __ ldr(rscratch2, mdo_addr);
2976         __ eor(tmp, tmp, rscratch2);
2977         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2978         // klass seen before, nothing to do. The unknown bit may have been
2979         // set already but no need to check.
2980         __ cbz(rscratch1, next);
2981 
2982         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2983 
2984         if (TypeEntries::is_type_none(current_klass)) {
2985           __ cbz(rscratch2, none);
2986           __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2987           __ br(Assembler::EQ, none);
2988           // There is a chance that the checks above (re-reading profiling
2989           // data from memory) fail if another thread has just set the
2990           // profiling to this obj's klass
2991           __ dmb(Assembler::ISHLD);
2992           __ ldr(rscratch2, mdo_addr);
2993           __ eor(tmp, tmp, rscratch2);
2994           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2995           __ cbz(rscratch1, next);
2996         }
2997       } else {
2998         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2999                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3000 
3001         __ ldr(tmp, mdo_addr);
3002         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
3003       }
3004 
3005       // different than before. Cannot keep accurate profile.
3006       __ ldr(rscratch2, mdo_addr);
3007       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
3008       __ str(rscratch2, mdo_addr);
3009 
3010       if (TypeEntries::is_type_none(current_klass)) {
3011         __ b(next);
3012 
3013         __ bind(none);
3014         // first time here. Set profile type.
3015         __ str(tmp, mdo_addr);
3016       }
3017     } else {
3018       // There's a single possible klass at this profile point
3019       assert(exact_klass != NULL, "should be");
3020       if (TypeEntries::is_type_none(current_klass)) {
3021         __ mov_metadata(tmp, exact_klass->constant_encoding());
3022         __ ldr(rscratch2, mdo_addr);
3023         __ eor(tmp, tmp, rscratch2);
3024         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
3025         __ cbz(rscratch1, next);
3026 #ifdef ASSERT
3027         {
3028           Label ok;
3029           __ ldr(rscratch1, mdo_addr);
3030           __ cbz(rscratch1, ok);
3031           __ cmp(rscratch1, (u1)TypeEntries::null_seen);
3032           __ br(Assembler::EQ, ok);
3033           // may have been set by another thread
3034           __ dmb(Assembler::ISHLD);
3035           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
3036           __ ldr(rscratch2, mdo_addr);
3037           __ eor(rscratch2, rscratch1, rscratch2);
3038           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
3039           __ cbz(rscratch2, ok);
3040 
3041           __ stop("unexpected profiling mismatch");
3042           __ bind(ok);
3043         }
3044 #endif
3045         // first time here. Set profile type.
3046         __ str(tmp, mdo_addr);
3047       } else {
3048         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3049                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3050 
3051         __ ldr(tmp, mdo_addr);
3052         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
3053 
3054         __ orr(tmp, tmp, TypeEntries::type_unknown);
3055         __ str(tmp, mdo_addr);
3056         // FIXME: Write barrier needed here?
3057       }
3058     }
3059 
3060     __ bind(next);
3061   }
3062   COMMENT("} emit_profile_type");
3063 }
3064 
3065 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3066   Register obj = op->obj()->as_register();
3067   Register tmp = op->tmp()->as_pointer_register();
3068   bool not_null = op->not_null();
3069   int flag = op->flag();
3070 
3071   Label not_inline_type;
3072   if (!not_null) {
3073     __ cbz(obj, not_inline_type);
3074   }
3075 
3076   __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3077 
3078   Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), rscratch2);
3079   __ ldrb(rscratch1, mdo_addr);
3080   __ orr(rscratch1, rscratch1, flag);
3081   __ strb(rscratch1, mdo_addr);
3082 
3083   __ bind(not_inline_type);
3084 }
3085 
3086 void LIR_Assembler::align_backward_branch_target() {
3087 }
3088 
3089 
3090 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3091   // tmp must be unused
3092   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
3093 
3094   if (left->is_single_cpu()) {
3095     assert(dest->is_single_cpu(), "expect single result reg");
3096     __ negw(dest->as_register(), left->as_register());
3097   } else if (left->is_double_cpu()) {
3098     assert(dest->is_double_cpu(), "expect double result reg");
3099     __ neg(dest->as_register_lo(), left->as_register_lo());
3100   } else if (left->is_single_fpu()) {
3101     assert(dest->is_single_fpu(), "expect single float result reg");
3102     __ fnegs(dest->as_float_reg(), left->as_float_reg());
3103   } else {
3104     assert(left->is_double_fpu(), "expect double float operand reg");
3105     assert(dest->is_double_fpu(), "expect double float result reg");
3106     __ fnegd(dest->as_double_reg(), left->as_double_reg());
3107   }
3108 }
3109 
3110 
3111 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3112   if (patch_code != lir_patch_none) {
3113     deoptimize_trap(info);
3114     return;
3115   }
3116 
3117   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
3118 }
3119 
3120 
3121 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3122   assert(!tmp->is_valid(), "don't need temporary");
3123 
3124   CodeBlob *cb = CodeCache::find_blob(dest);
3125   if (cb) {
3126     __ far_call(RuntimeAddress(dest));
3127   } else {
3128     __ mov(rscratch1, RuntimeAddress(dest));
3129     __ blr(rscratch1);
3130   }
3131 
3132   if (info != NULL) {
3133     add_call_info_here(info);
3134   }
3135   __ post_call_nop();
3136 }
3137 
3138 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3139   if (dest->is_address() || src->is_address()) {
3140     move_op(src, dest, type, lir_patch_none, info,
3141             /*pop_fpu_stack*/false, /*wide*/false);
3142   } else {
3143     ShouldNotReachHere();
3144   }
3145 }
3146 
3147 #ifdef ASSERT
3148 // emit run-time assertion
3149 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3150   assert(op->code() == lir_assert, "must be");
3151 
3152   if (op->in_opr1()->is_valid()) {
3153     assert(op->in_opr2()->is_valid(), "both operands must be valid");
3154     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3155   } else {
3156     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3157     assert(op->condition() == lir_cond_always, "no other conditions allowed");
3158   }
3159 
3160   Label ok;
3161   if (op->condition() != lir_cond_always) {
3162     Assembler::Condition acond = Assembler::AL;
3163     switch (op->condition()) {
3164       case lir_cond_equal:        acond = Assembler::EQ;  break;
3165       case lir_cond_notEqual:     acond = Assembler::NE;  break;
3166       case lir_cond_less:         acond = Assembler::LT;  break;
3167       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
3168       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
3169       case lir_cond_greater:      acond = Assembler::GT;  break;
3170       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
3171       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
3172       default:                    ShouldNotReachHere();
3173     }
3174     __ br(acond, ok);
3175   }
3176   if (op->halt()) {
3177     const char* str = __ code_string(op->msg());
3178     __ stop(str);
3179   } else {
3180     breakpoint();
3181   }
3182   __ bind(ok);
3183 }
3184 #endif
3185 
3186 #ifndef PRODUCT
3187 #define COMMENT(x)   do { __ block_comment(x); } while (0)
3188 #else
3189 #define COMMENT(x)
3190 #endif
3191 
3192 void LIR_Assembler::membar() {
3193   COMMENT("membar");
3194   __ membar(MacroAssembler::AnyAny);
3195 }
3196 
3197 void LIR_Assembler::membar_acquire() {
3198   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3199 }
3200 
3201 void LIR_Assembler::membar_release() {
3202   __ membar(Assembler::LoadStore|Assembler::StoreStore);
3203 }
3204 
3205 void LIR_Assembler::membar_loadload() {
3206   __ membar(Assembler::LoadLoad);
3207 }
3208 
3209 void LIR_Assembler::membar_storestore() {
3210   __ membar(MacroAssembler::StoreStore);
3211 }
3212 
3213 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3214 
3215 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3216 
3217 void LIR_Assembler::on_spin_wait() {
3218   __ spin_wait();
3219 }
3220 
3221 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3222   __ mov(result_reg->as_register(), rthread);
3223 }
3224 
3225 void LIR_Assembler::check_orig_pc() {
3226   __ ldr(rscratch2, frame_map()->address_for_orig_pc_addr());
3227   __ cmp(rscratch2, (u1)NULL_WORD);
3228 }
3229 
3230 void LIR_Assembler::peephole(LIR_List *lir) {
3231 #if 0
3232   if (tableswitch_count >= max_tableswitches)
3233     return;
3234 
3235   /*
3236     This finite-state automaton recognizes sequences of compare-and-
3237     branch instructions.  We will turn them into a tableswitch.  You
3238     could argue that C1 really shouldn't be doing this sort of
3239     optimization, but without it the code is really horrible.
3240   */
3241 
3242   enum { start_s, cmp1_s, beq_s, cmp_s } state;
3243   int first_key, last_key = -2147483648;
3244   int next_key = 0;
3245   int start_insn = -1;
3246   int last_insn = -1;
3247   Register reg = noreg;
3248   LIR_Opr reg_opr;
3249   state = start_s;
3250 
3251   LIR_OpList* inst = lir->instructions_list();
3252   for (int i = 0; i < inst->length(); i++) {
3253     LIR_Op* op = inst->at(i);
3254     switch (state) {
3255     case start_s:
3256       first_key = -1;
3257       start_insn = i;
3258       switch (op->code()) {
3259       case lir_cmp:
3260         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3261         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3262         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3263             && opr2->is_constant()
3264             && opr2->type() == T_INT) {
3265           reg_opr = opr1;
3266           reg = opr1->as_register();
3267           first_key = opr2->as_constant_ptr()->as_jint();
3268           next_key = first_key + 1;
3269           state = cmp_s;
3270           goto next_state;
3271         }
3272         break;
3273       }
3274       break;
3275     case cmp_s:
3276       switch (op->code()) {
3277       case lir_branch:
3278         if (op->as_OpBranch()->cond() == lir_cond_equal) {
3279           state = beq_s;
3280           last_insn = i;
3281           goto next_state;
3282         }
3283       }
3284       state = start_s;
3285       break;
3286     case beq_s:
3287       switch (op->code()) {
3288       case lir_cmp: {
3289         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3290         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3291         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3292             && opr1->as_register() == reg
3293             && opr2->is_constant()
3294             && opr2->type() == T_INT
3295             && opr2->as_constant_ptr()->as_jint() == next_key) {
3296           last_key = next_key;
3297           next_key++;
3298           state = cmp_s;
3299           goto next_state;
3300         }
3301       }
3302       }
3303       last_key = next_key;
3304       state = start_s;
3305       break;
3306     default:
3307       assert(false, "impossible state");
3308     }
3309     if (state == start_s) {
3310       if (first_key < last_key - 5L && reg != noreg) {
3311         {
3312           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3313           //        reg->encoding(),
3314           //        start_insn, first_key, last_key);
3315           //   for (int i = 0; i < inst->length(); i++) {
3316           //     inst->at(i)->print();
3317           //     tty->print("\n");
3318           //   }
3319           //   tty->print("\n");
3320         }
3321 
3322         struct tableswitch *sw = &switches[tableswitch_count];
3323         sw->_insn_index = start_insn, sw->_first_key = first_key,
3324           sw->_last_key = last_key, sw->_reg = reg;
3325         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3326         {
3327           // Insert the new table of branches
3328           int offset = last_insn;
3329           for (int n = first_key; n < last_key; n++) {
3330             inst->insert_before
3331               (last_insn + 1,
3332                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3333                                 inst->at(offset)->as_OpBranch()->label()));
3334             offset -= 2, i++;
3335           }
3336         }
3337         // Delete all the old compare-and-branch instructions
3338         for (int n = first_key; n < last_key; n++) {
3339           inst->remove_at(start_insn);
3340           inst->remove_at(start_insn);
3341         }
3342         // Insert the tableswitch instruction
3343         inst->insert_before(start_insn,
3344                             new LIR_Op2(lir_cmp, lir_cond_always,
3345                                         LIR_OprFact::intConst(tableswitch_count),
3346                                         reg_opr));
3347         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3348         tableswitch_count++;
3349       }
3350       reg = noreg;
3351       last_key = -2147483648;
3352     }
3353   next_state:
3354     ;
3355   }
3356 #endif
3357 }
3358 
3359 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3360   Address addr = as_Address(src->as_address_ptr());
3361   BasicType type = src->type();
3362   bool is_oop = is_reference_type(type);
3363 
3364   void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3365   void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3366 
3367   switch(type) {
3368   case T_INT:
3369     xchg = &MacroAssembler::atomic_xchgalw;
3370     add = &MacroAssembler::atomic_addalw;
3371     break;
3372   case T_LONG:
3373     xchg = &MacroAssembler::atomic_xchgal;
3374     add = &MacroAssembler::atomic_addal;
3375     break;
3376   case T_PRIMITIVE_OBJECT:
3377   case T_OBJECT:
3378   case T_ARRAY:
3379     if (UseCompressedOops) {
3380       xchg = &MacroAssembler::atomic_xchgalw;
3381       add = &MacroAssembler::atomic_addalw;
3382     } else {
3383       xchg = &MacroAssembler::atomic_xchgal;
3384       add = &MacroAssembler::atomic_addal;
3385     }
3386     break;
3387   default:
3388     ShouldNotReachHere();
3389     xchg = &MacroAssembler::atomic_xchgal;
3390     add = &MacroAssembler::atomic_addal; // unreachable
3391   }
3392 
3393   switch (code) {
3394   case lir_xadd:
3395     {
3396       RegisterOrConstant inc;
3397       Register tmp = as_reg(tmp_op);
3398       Register dst = as_reg(dest);
3399       if (data->is_constant()) {
3400         inc = RegisterOrConstant(as_long(data));
3401         assert_different_registers(dst, addr.base(), tmp,
3402                                    rscratch1, rscratch2);
3403       } else {
3404         inc = RegisterOrConstant(as_reg(data));
3405         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3406                                    rscratch1, rscratch2);
3407       }
3408       __ lea(tmp, addr);
3409       (_masm->*add)(dst, inc, tmp);
3410       break;
3411     }
3412   case lir_xchg:
3413     {
3414       Register tmp = tmp_op->as_register();
3415       Register obj = as_reg(data);
3416       Register dst = as_reg(dest);
3417       if (is_oop && UseCompressedOops) {
3418         __ encode_heap_oop(rscratch2, obj);
3419         obj = rscratch2;
3420       }
3421       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3422       __ lea(tmp, addr);
3423       (_masm->*xchg)(dst, obj, tmp);
3424       if (is_oop && UseCompressedOops) {
3425         __ decode_heap_oop(dst);
3426       }
3427     }
3428     break;
3429   default:
3430     ShouldNotReachHere();
3431   }
3432   __ membar(__ AnyAny);
3433 }
3434 
3435 #undef __