1 /*
   2  * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Compilation.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_MacroAssembler.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArrayKlass.hpp"
  36 #include "ci/ciInstance.hpp"
  37 #include "code/compiledIC.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "gc/shared/gc_globals.hpp"
  40 #include "nativeInst_aarch64.hpp"
  41 #include "oops/compressedKlass.hpp"
  42 #include "oops/objArrayKlass.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 #include "utilities/powerOfTwo.hpp"
  47 #include "vmreg_aarch64.inline.hpp"
  48 
  49 
  50 #ifndef PRODUCT
  51 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  52 #else
  53 #define COMMENT(x)
  54 #endif
  55 
  56 NEEDS_CLEANUP // remove this definitions ?
  57 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  58 const Register SYNC_header = r0;   // synchronization header
  59 const Register SHIFT_count = r0;   // where count for shift operations must be
  60 
  61 #define __ _masm->
  62 
  63 
  64 static void select_different_registers(Register preserve,
  65                                        Register extra,
  66                                        Register &tmp1,
  67                                        Register &tmp2) {
  68   if (tmp1 == preserve) {
  69     assert_different_registers(tmp1, tmp2, extra);
  70     tmp1 = extra;
  71   } else if (tmp2 == preserve) {
  72     assert_different_registers(tmp1, tmp2, extra);
  73     tmp2 = extra;
  74   }
  75   assert_different_registers(preserve, tmp1, tmp2);
  76 }
  77 
  78 
  79 
  80 static void select_different_registers(Register preserve,
  81                                        Register extra,
  82                                        Register &tmp1,
  83                                        Register &tmp2,
  84                                        Register &tmp3) {
  85   if (tmp1 == preserve) {
  86     assert_different_registers(tmp1, tmp2, tmp3, extra);
  87     tmp1 = extra;
  88   } else if (tmp2 == preserve) {
  89     assert_different_registers(tmp1, tmp2, tmp3, extra);
  90     tmp2 = extra;
  91   } else if (tmp3 == preserve) {
  92     assert_different_registers(tmp1, tmp2, tmp3, extra);
  93     tmp3 = extra;
  94   }
  95   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  96 }
  97 
  98 
  99 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
 100 
 101 
 102 LIR_Opr LIR_Assembler::receiverOpr() {
 103   return FrameMap::receiver_opr;
 104 }
 105 
 106 LIR_Opr LIR_Assembler::osrBufferPointer() {
 107   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 108 }
 109 
 110 //--------------fpu register translations-----------------------
 111 
 112 
 113 address LIR_Assembler::float_constant(float f) {
 114   address const_addr = __ float_constant(f);
 115   if (const_addr == NULL) {
 116     bailout("const section overflow");
 117     return __ code()->consts()->start();
 118   } else {
 119     return const_addr;
 120   }
 121 }
 122 
 123 
 124 address LIR_Assembler::double_constant(double d) {
 125   address const_addr = __ double_constant(d);
 126   if (const_addr == NULL) {
 127     bailout("const section overflow");
 128     return __ code()->consts()->start();
 129   } else {
 130     return const_addr;
 131   }
 132 }
 133 
 134 address LIR_Assembler::int_constant(jlong n) {
 135   address const_addr = __ long_constant(n);
 136   if (const_addr == NULL) {
 137     bailout("const section overflow");
 138     return __ code()->consts()->start();
 139   } else {
 140     return const_addr;
 141   }
 142 }
 143 
 144 void LIR_Assembler::breakpoint() { Unimplemented(); }
 145 
 146 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 147 
 148 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 149 
 150 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
 151 //-------------------------------------------
 152 
 153 static Register as_reg(LIR_Opr op) {
 154   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 155 }
 156 
 157 static jlong as_long(LIR_Opr data) {
 158   jlong result;
 159   switch (data->type()) {
 160   case T_INT:
 161     result = (data->as_jint());
 162     break;
 163   case T_LONG:
 164     result = (data->as_jlong());
 165     break;
 166   default:
 167     ShouldNotReachHere();
 168     result = 0;  // unreachable
 169   }
 170   return result;
 171 }
 172 
 173 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 174   Register base = addr->base()->as_pointer_register();
 175   LIR_Opr opr = addr->index();
 176   if (opr->is_cpu_register()) {
 177     Register index;
 178     if (opr->is_single_cpu())
 179       index = opr->as_register();
 180     else
 181       index = opr->as_register_lo();
 182     assert(addr->disp() == 0, "must be");
 183     switch(opr->type()) {
 184       case T_INT:
 185         return Address(base, index, Address::sxtw(addr->scale()));
 186       case T_LONG:
 187         return Address(base, index, Address::lsl(addr->scale()));
 188       default:
 189         ShouldNotReachHere();
 190       }
 191   } else {
 192     assert(addr->scale() == 0,
 193            "expected for immediate operand, was: %d", addr->scale());
 194     ptrdiff_t offset = ptrdiff_t(addr->disp());
 195     // NOTE: Does not handle any 16 byte vector access.
 196     const uint type_size = type2aelembytes(addr->type(), true);
 197     return __ legitimize_address(Address(base, offset), type_size, tmp);
 198   }
 199   return Address();
 200 }
 201 
 202 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 203   ShouldNotReachHere();
 204   return Address();
 205 }
 206 
 207 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 208   return as_Address(addr, rscratch1);
 209 }
 210 
 211 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 212   return as_Address(addr, rscratch1);  // Ouch
 213   // FIXME: This needs to be much more clever.  See x86.
 214 }
 215 
 216 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
 217 // not encodable as a base + (immediate) offset, generate an explicit address
 218 // calculation to hold the address in a temporary register.
 219 Address LIR_Assembler::stack_slot_address(int index, uint size, Register tmp, int adjust) {
 220   precond(size == 4 || size == 8);
 221   Address addr = frame_map()->address_for_slot(index, adjust);
 222   precond(addr.getMode() == Address::base_plus_offset);
 223   precond(addr.base() == sp);
 224   precond(addr.offset() > 0);
 225   uint mask = size - 1;
 226   assert((addr.offset() & mask) == 0, "scaled offsets only");
 227   return __ legitimize_address(addr, size, tmp);
 228 }
 229 
 230 void LIR_Assembler::osr_entry() {
 231   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 232   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 233   ValueStack* entry_state = osr_entry->state();
 234   int number_of_locks = entry_state->locks_size();
 235 
 236   // we jump here if osr happens with the interpreter
 237   // state set up to continue at the beginning of the
 238   // loop that triggered osr - in particular, we have
 239   // the following registers setup:
 240   //
 241   // r2: osr buffer
 242   //
 243 
 244   // build frame
 245   ciMethod* m = compilation()->method();
 246   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes(), compilation()->max_monitors());
 247 
 248   // OSR buffer is
 249   //
 250   // locals[nlocals-1..0]
 251   // monitors[0..number_of_locks]
 252   //
 253   // locals is a direct copy of the interpreter frame so in the osr buffer
 254   // so first slot in the local array is the last local from the interpreter
 255   // and last slot is local[0] (receiver) from the interpreter
 256   //
 257   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 258   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 259   // in the interpreter frame (the method lock if a sync method)
 260 
 261   // Initialize monitors in the compiled activation.
 262   //   r2: pointer to osr buffer
 263   //
 264   // All other registers are dead at this point and the locals will be
 265   // copied into place by code emitted in the IR.
 266 
 267   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 268   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 269     int monitor_offset = BytesPerWord * method()->max_locals() +
 270       (2 * BytesPerWord) * (number_of_locks - 1);
 271     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 272     // the OSR buffer using 2 word entries: first the lock and then
 273     // the oop.
 274     for (int i = 0; i < number_of_locks; i++) {
 275       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 276 #ifdef ASSERT
 277       // verify the interpreter's monitor has a non-null object
 278       {
 279         Label L;
 280         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 281         __ cbnz(rscratch1, L);
 282         __ stop("locked object is NULL");
 283         __ bind(L);
 284       }
 285 #endif
 286       __ ldp(r19, r20, Address(OSR_buf, slot_offset));
 287       __ str(r19, frame_map()->address_for_monitor_lock(i));
 288       __ str(r20, frame_map()->address_for_monitor_object(i));
 289     }
 290   }
 291 }
 292 
 293 
 294 // inline cache check; done before the frame is built.
 295 int LIR_Assembler::check_icache() {
 296   Register receiver = FrameMap::receiver_opr->as_register();
 297   Register ic_klass = IC_Klass;
 298   int start_offset = __ offset();
 299   __ inline_cache_check(receiver, ic_klass);
 300 
 301   // if icache check fails, then jump to runtime routine
 302   // Note: RECEIVER must still contain the receiver!
 303   Label dont;
 304   __ br(Assembler::EQ, dont);
 305   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 306 
 307   // We align the verified entry point unless the method body
 308   // (including its inline cache check) will fit in a single 64-byte
 309   // icache line.
 310   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 311     // force alignment after the cache check.
 312     __ align(CodeEntryAlignment);
 313   }
 314 
 315   __ bind(dont);
 316   return start_offset;
 317 }
 318 
 319 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 320   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 321   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 322 
 323   Label L_skip_barrier;
 324 
 325   __ mov_metadata(rscratch2, method->holder()->constant_encoding());
 326   __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier /*L_fast_path*/);
 327   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 328   __ bind(L_skip_barrier);
 329 }
 330 
 331 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 332   if (o == NULL) {
 333     __ mov(reg, zr);
 334   } else {
 335     __ movoop(reg, o);
 336   }
 337 }
 338 
 339 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 340   address target = NULL;
 341   relocInfo::relocType reloc_type = relocInfo::none;
 342 
 343   switch (patching_id(info)) {
 344   case PatchingStub::access_field_id:
 345     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 346     reloc_type = relocInfo::section_word_type;
 347     break;
 348   case PatchingStub::load_klass_id:
 349     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 350     reloc_type = relocInfo::metadata_type;
 351     break;
 352   case PatchingStub::load_mirror_id:
 353     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 354     reloc_type = relocInfo::oop_type;
 355     break;
 356   case PatchingStub::load_appendix_id:
 357     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 358     reloc_type = relocInfo::oop_type;
 359     break;
 360   default: ShouldNotReachHere();
 361   }
 362 
 363   __ far_call(RuntimeAddress(target));
 364   add_call_info_here(info);
 365 }
 366 
 367 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 368   deoptimize_trap(info);
 369 }
 370 
 371 
 372 // This specifies the rsp decrement needed to build the frame
 373 int LIR_Assembler::initial_frame_size_in_bytes() const {
 374   // if rounding, must let FrameMap know!
 375 
 376   return in_bytes(frame_map()->framesize_in_bytes());
 377 }
 378 
 379 
 380 int LIR_Assembler::emit_exception_handler() {
 381   // generate code for exception handler
 382   address handler_base = __ start_a_stub(exception_handler_size());
 383   if (handler_base == NULL) {
 384     // not enough space left for the handler
 385     bailout("exception handler overflow");
 386     return -1;
 387   }
 388 
 389   int offset = code_offset();
 390 
 391   // the exception oop and pc are in r0, and r3
 392   // no other registers need to be preserved, so invalidate them
 393   __ invalidate_registers(false, true, true, false, true, true);
 394 
 395   // check that there is really an exception
 396   __ verify_not_null_oop(r0);
 397 
 398   // search an exception handler (r0: exception oop, r3: throwing pc)
 399   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 400   __ should_not_reach_here();
 401   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 402   __ end_a_stub();
 403 
 404   return offset;
 405 }
 406 
 407 
 408 // Emit the code to remove the frame from the stack in the exception
 409 // unwind path.
 410 int LIR_Assembler::emit_unwind_handler() {
 411 #ifndef PRODUCT
 412   if (CommentedAssembly) {
 413     _masm->block_comment("Unwind handler");
 414   }
 415 #endif
 416 
 417   int offset = code_offset();
 418 
 419   // Fetch the exception from TLS and clear out exception related thread state
 420   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 421   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 422   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 423 
 424   __ bind(_unwind_handler_entry);
 425   __ verify_not_null_oop(r0);
 426   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 427     __ mov(r19, r0);  // Preserve the exception
 428   }
 429 
 430   // Perform needed unlocking
 431   MonitorExitStub* stub = NULL;
 432   if (method()->is_synchronized()) {
 433     monitor_address(0, FrameMap::r0_opr);
 434     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
 435     if (UseHeavyMonitors) {
 436       __ b(*stub->entry());
 437     } else {
 438       __ unlock_object(r5, r4, r0, *stub->entry());
 439     }
 440     __ bind(*stub->continuation());
 441   }
 442 
 443   if (compilation()->env()->dtrace_method_probes()) {
 444     __ mov(c_rarg0, rthread);
 445     __ mov_metadata(c_rarg1, method()->constant_encoding());
 446     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 447   }
 448 
 449   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 450     __ mov(r0, r19);  // Restore the exception
 451   }
 452 
 453   // remove the activation and dispatch to the unwind handler
 454   __ block_comment("remove_frame and dispatch to the unwind handler");
 455   __ remove_frame(initial_frame_size_in_bytes());
 456   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 457 
 458   // Emit the slow path assembly
 459   if (stub != NULL) {
 460     stub->emit_code(this);
 461   }
 462 
 463   return offset;
 464 }
 465 
 466 
 467 int LIR_Assembler::emit_deopt_handler() {
 468   // generate code for exception handler
 469   address handler_base = __ start_a_stub(deopt_handler_size());
 470   if (handler_base == NULL) {
 471     // not enough space left for the handler
 472     bailout("deopt handler overflow");
 473     return -1;
 474   }
 475 
 476   int offset = code_offset();
 477 
 478   __ adr(lr, pc());
 479   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 480   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 481   __ end_a_stub();
 482 
 483   return offset;
 484 }
 485 
 486 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 487   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 488   int pc_offset = code_offset();
 489   flush_debug_info(pc_offset);
 490   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 491   if (info->exception_handlers() != NULL) {
 492     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 493   }
 494 }
 495 
 496 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 497   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 498 
 499   // Pop the stack before the safepoint code
 500   __ remove_frame(initial_frame_size_in_bytes());
 501 
 502   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 503     __ reserved_stack_check();
 504   }
 505 
 506   code_stub->set_safepoint_offset(__ offset());
 507   __ relocate(relocInfo::poll_return_type);
 508   __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
 509   __ ret(lr);
 510 }
 511 
 512 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 513   guarantee(info != NULL, "Shouldn't be NULL");
 514   __ get_polling_page(rscratch1, relocInfo::poll_type);
 515   add_debug_info_for_branch(info);  // This isn't just debug info:
 516                                     // it's the oop map
 517   __ read_polling_page(rscratch1, relocInfo::poll_type);
 518   return __ offset();
 519 }
 520 
 521 
 522 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 523   if (from_reg == r31_sp)
 524     from_reg = sp;
 525   if (to_reg == r31_sp)
 526     to_reg = sp;
 527   __ mov(to_reg, from_reg);
 528 }
 529 
 530 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 531 
 532 
 533 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 534   assert(src->is_constant(), "should not call otherwise");
 535   assert(dest->is_register(), "should not call otherwise");
 536   LIR_Const* c = src->as_constant_ptr();
 537 
 538   switch (c->type()) {
 539     case T_INT: {
 540       assert(patch_code == lir_patch_none, "no patching handled here");
 541       __ movw(dest->as_register(), c->as_jint());
 542       break;
 543     }
 544 
 545     case T_ADDRESS: {
 546       assert(patch_code == lir_patch_none, "no patching handled here");
 547       __ mov(dest->as_register(), c->as_jint());
 548       break;
 549     }
 550 
 551     case T_LONG: {
 552       assert(patch_code == lir_patch_none, "no patching handled here");
 553       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
 554       break;
 555     }
 556 
 557     case T_OBJECT: {
 558         if (patch_code == lir_patch_none) {
 559           jobject2reg(c->as_jobject(), dest->as_register());
 560         } else {
 561           jobject2reg_with_patching(dest->as_register(), info);
 562         }
 563       break;
 564     }
 565 
 566     case T_METADATA: {
 567       if (patch_code != lir_patch_none) {
 568         klass2reg_with_patching(dest->as_register(), info);
 569       } else {
 570         __ mov_metadata(dest->as_register(), c->as_metadata());
 571       }
 572       break;
 573     }
 574 
 575     case T_FLOAT: {
 576       if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 577         __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
 578       } else {
 579         __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 580         __ ldrs(dest->as_float_reg(), Address(rscratch1));
 581       }
 582       break;
 583     }
 584 
 585     case T_DOUBLE: {
 586       if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
 587         __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
 588       } else {
 589         __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 590         __ ldrd(dest->as_double_reg(), Address(rscratch1));
 591       }
 592       break;
 593     }
 594 
 595     default:
 596       ShouldNotReachHere();
 597   }
 598 }
 599 
 600 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 601   LIR_Const* c = src->as_constant_ptr();
 602   switch (c->type()) {
 603   case T_OBJECT:
 604     {
 605       if (! c->as_jobject())
 606         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 607       else {
 608         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 609         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 610       }
 611     }
 612     break;
 613   case T_ADDRESS:
 614     {
 615       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 616       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 617     }
 618   case T_INT:
 619   case T_FLOAT:
 620     {
 621       Register reg = zr;
 622       if (c->as_jint_bits() == 0)
 623         __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 624       else {
 625         __ movw(rscratch1, c->as_jint_bits());
 626         __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 627       }
 628     }
 629     break;
 630   case T_LONG:
 631   case T_DOUBLE:
 632     {
 633       Register reg = zr;
 634       if (c->as_jlong_bits() == 0)
 635         __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 636                                                  lo_word_offset_in_bytes));
 637       else {
 638         __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
 639         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 640                                                         lo_word_offset_in_bytes));
 641       }
 642     }
 643     break;
 644   default:
 645     ShouldNotReachHere();
 646   }
 647 }
 648 
 649 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 650   assert(src->is_constant(), "should not call otherwise");
 651   LIR_Const* c = src->as_constant_ptr();
 652   LIR_Address* to_addr = dest->as_address_ptr();
 653 
 654   void (Assembler::* insn)(Register Rt, const Address &adr);
 655 
 656   switch (type) {
 657   case T_ADDRESS:
 658     assert(c->as_jint() == 0, "should be");
 659     insn = &Assembler::str;
 660     break;
 661   case T_LONG:
 662     assert(c->as_jlong() == 0, "should be");
 663     insn = &Assembler::str;
 664     break;
 665   case T_INT:
 666     assert(c->as_jint() == 0, "should be");
 667     insn = &Assembler::strw;
 668     break;
 669   case T_OBJECT:
 670   case T_ARRAY:
 671     assert(c->as_jobject() == 0, "should be");
 672     if (UseCompressedOops && !wide) {
 673       insn = &Assembler::strw;
 674     } else {
 675       insn = &Assembler::str;
 676     }
 677     break;
 678   case T_CHAR:
 679   case T_SHORT:
 680     assert(c->as_jint() == 0, "should be");
 681     insn = &Assembler::strh;
 682     break;
 683   case T_BOOLEAN:
 684   case T_BYTE:
 685     assert(c->as_jint() == 0, "should be");
 686     insn = &Assembler::strb;
 687     break;
 688   default:
 689     ShouldNotReachHere();
 690     insn = &Assembler::str;  // unreachable
 691   }
 692 
 693   if (info) add_debug_info_for_null_check_here(info);
 694   (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
 695 }
 696 
 697 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 698   assert(src->is_register(), "should not call otherwise");
 699   assert(dest->is_register(), "should not call otherwise");
 700 
 701   // move between cpu-registers
 702   if (dest->is_single_cpu()) {
 703     if (src->type() == T_LONG) {
 704       // Can do LONG -> OBJECT
 705       move_regs(src->as_register_lo(), dest->as_register());
 706       return;
 707     }
 708     assert(src->is_single_cpu(), "must match");
 709     if (src->type() == T_OBJECT) {
 710       __ verify_oop(src->as_register());
 711     }
 712     move_regs(src->as_register(), dest->as_register());
 713 
 714   } else if (dest->is_double_cpu()) {
 715     if (is_reference_type(src->type())) {
 716       // Surprising to me but we can see move of a long to t_object
 717       __ verify_oop(src->as_register());
 718       move_regs(src->as_register(), dest->as_register_lo());
 719       return;
 720     }
 721     assert(src->is_double_cpu(), "must match");
 722     Register f_lo = src->as_register_lo();
 723     Register f_hi = src->as_register_hi();
 724     Register t_lo = dest->as_register_lo();
 725     Register t_hi = dest->as_register_hi();
 726     assert(f_hi == f_lo, "must be same");
 727     assert(t_hi == t_lo, "must be same");
 728     move_regs(f_lo, t_lo);
 729 
 730   } else if (dest->is_single_fpu()) {
 731     __ fmovs(dest->as_float_reg(), src->as_float_reg());
 732 
 733   } else if (dest->is_double_fpu()) {
 734     __ fmovd(dest->as_double_reg(), src->as_double_reg());
 735 
 736   } else {
 737     ShouldNotReachHere();
 738   }
 739 }
 740 
 741 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 742   precond(src->is_register() && dest->is_stack());
 743 
 744   uint const c_sz32 = sizeof(uint32_t);
 745   uint const c_sz64 = sizeof(uint64_t);
 746 
 747   if (src->is_single_cpu()) {
 748     int index = dest->single_stack_ix();
 749     if (is_reference_type(type)) {
 750       __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 751       __ verify_oop(src->as_register());
 752     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 753       __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 754     } else {
 755       __ strw(src->as_register(), stack_slot_address(index, c_sz32, rscratch1));
 756     }
 757 
 758   } else if (src->is_double_cpu()) {
 759     int index = dest->double_stack_ix();
 760     Address dest_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
 761     __ str(src->as_register_lo(), dest_addr_LO);
 762 
 763   } else if (src->is_single_fpu()) {
 764     int index = dest->single_stack_ix();
 765     __ strs(src->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
 766 
 767   } else if (src->is_double_fpu()) {
 768     int index = dest->double_stack_ix();
 769     __ strd(src->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
 770 
 771   } else {
 772     ShouldNotReachHere();
 773   }
 774 }
 775 
 776 
 777 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
 778   LIR_Address* to_addr = dest->as_address_ptr();
 779   PatchingStub* patch = NULL;
 780   Register compressed_src = rscratch1;
 781 
 782   if (patch_code != lir_patch_none) {
 783     deoptimize_trap(info);
 784     return;
 785   }
 786 
 787   if (is_reference_type(type)) {
 788     __ verify_oop(src->as_register());
 789 
 790     if (UseCompressedOops && !wide) {
 791       __ encode_heap_oop(compressed_src, src->as_register());
 792     } else {
 793       compressed_src = src->as_register();
 794     }
 795   }
 796 
 797   int null_check_here = code_offset();
 798   switch (type) {
 799     case T_FLOAT: {
 800       __ strs(src->as_float_reg(), as_Address(to_addr));
 801       break;
 802     }
 803 
 804     case T_DOUBLE: {
 805       __ strd(src->as_double_reg(), as_Address(to_addr));
 806       break;
 807     }
 808 
 809     case T_ARRAY:   // fall through
 810     case T_OBJECT:  // fall through
 811       if (UseCompressedOops && !wide) {
 812         __ strw(compressed_src, as_Address(to_addr, rscratch2));
 813       } else {
 814          __ str(compressed_src, as_Address(to_addr));
 815       }
 816       break;
 817     case T_METADATA:
 818       // We get here to store a method pointer to the stack to pass to
 819       // a dtrace runtime call. This can't work on 64 bit with
 820       // compressed klass ptrs: T_METADATA can be a compressed klass
 821       // ptr or a 64 bit method pointer.
 822       ShouldNotReachHere();
 823       __ str(src->as_register(), as_Address(to_addr));
 824       break;
 825     case T_ADDRESS:
 826       __ str(src->as_register(), as_Address(to_addr));
 827       break;
 828     case T_INT:
 829       __ strw(src->as_register(), as_Address(to_addr));
 830       break;
 831 
 832     case T_LONG: {
 833       __ str(src->as_register_lo(), as_Address_lo(to_addr));
 834       break;
 835     }
 836 
 837     case T_BYTE:    // fall through
 838     case T_BOOLEAN: {
 839       __ strb(src->as_register(), as_Address(to_addr));
 840       break;
 841     }
 842 
 843     case T_CHAR:    // fall through
 844     case T_SHORT:
 845       __ strh(src->as_register(), as_Address(to_addr));
 846       break;
 847 
 848     default:
 849       ShouldNotReachHere();
 850   }
 851   if (info != NULL) {
 852     add_debug_info_for_null_check(null_check_here, info);
 853   }
 854 }
 855 
 856 
 857 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 858   precond(src->is_stack() && dest->is_register());
 859 
 860   uint const c_sz32 = sizeof(uint32_t);
 861   uint const c_sz64 = sizeof(uint64_t);
 862 
 863   if (dest->is_single_cpu()) {
 864     int index = src->single_stack_ix();
 865     if (is_reference_type(type)) {
 866       __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 867       __ verify_oop(dest->as_register());
 868     } else if (type == T_METADATA || type == T_ADDRESS) {
 869       __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 870     } else {
 871       __ ldrw(dest->as_register(), stack_slot_address(index, c_sz32, rscratch1));
 872     }
 873 
 874   } else if (dest->is_double_cpu()) {
 875     int index = src->double_stack_ix();
 876     Address src_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
 877     __ ldr(dest->as_register_lo(), src_addr_LO);
 878 
 879   } else if (dest->is_single_fpu()) {
 880     int index = src->single_stack_ix();
 881     __ ldrs(dest->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
 882 
 883   } else if (dest->is_double_fpu()) {
 884     int index = src->double_stack_ix();
 885     __ ldrd(dest->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
 886 
 887   } else {
 888     ShouldNotReachHere();
 889   }
 890 }
 891 
 892 
 893 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 894   address target = NULL;
 895   relocInfo::relocType reloc_type = relocInfo::none;
 896 
 897   switch (patching_id(info)) {
 898   case PatchingStub::access_field_id:
 899     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 900     reloc_type = relocInfo::section_word_type;
 901     break;
 902   case PatchingStub::load_klass_id:
 903     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 904     reloc_type = relocInfo::metadata_type;
 905     break;
 906   case PatchingStub::load_mirror_id:
 907     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 908     reloc_type = relocInfo::oop_type;
 909     break;
 910   case PatchingStub::load_appendix_id:
 911     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 912     reloc_type = relocInfo::oop_type;
 913     break;
 914   default: ShouldNotReachHere();
 915   }
 916 
 917   __ far_call(RuntimeAddress(target));
 918   add_call_info_here(info);
 919 }
 920 
 921 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 922 
 923   LIR_Opr temp;
 924   if (type == T_LONG || type == T_DOUBLE)
 925     temp = FrameMap::rscratch1_long_opr;
 926   else
 927     temp = FrameMap::rscratch1_opr;
 928 
 929   stack2reg(src, temp, src->type());
 930   reg2stack(temp, dest, dest->type(), false);
 931 }
 932 
 933 
 934 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 935   LIR_Address* addr = src->as_address_ptr();
 936   LIR_Address* from_addr = src->as_address_ptr();
 937 
 938   if (addr->base()->type() == T_OBJECT) {
 939     __ verify_oop(addr->base()->as_pointer_register());
 940   }
 941 
 942   if (patch_code != lir_patch_none) {
 943     deoptimize_trap(info);
 944     return;
 945   }
 946 
 947   if (info != NULL) {
 948     add_debug_info_for_null_check_here(info);
 949   }
 950   int null_check_here = code_offset();
 951   switch (type) {
 952     case T_FLOAT: {
 953       __ ldrs(dest->as_float_reg(), as_Address(from_addr));
 954       break;
 955     }
 956 
 957     case T_DOUBLE: {
 958       __ ldrd(dest->as_double_reg(), as_Address(from_addr));
 959       break;
 960     }
 961 
 962     case T_ARRAY:   // fall through
 963     case T_OBJECT:  // fall through
 964       if (UseCompressedOops && !wide) {
 965         __ ldrw(dest->as_register(), as_Address(from_addr));
 966       } else {
 967         __ ldr(dest->as_register(), as_Address(from_addr));
 968       }
 969       break;
 970     case T_METADATA:
 971       // We get here to store a method pointer to the stack to pass to
 972       // a dtrace runtime call. This can't work on 64 bit with
 973       // compressed klass ptrs: T_METADATA can be a compressed klass
 974       // ptr or a 64 bit method pointer.
 975       ShouldNotReachHere();
 976       __ ldr(dest->as_register(), as_Address(from_addr));
 977       break;
 978     case T_ADDRESS:
 979       __ ldr(dest->as_register(), as_Address(from_addr));
 980       break;
 981     case T_INT:
 982       __ ldrw(dest->as_register(), as_Address(from_addr));
 983       break;
 984 
 985     case T_LONG: {
 986       __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
 987       break;
 988     }
 989 
 990     case T_BYTE:
 991       __ ldrsb(dest->as_register(), as_Address(from_addr));
 992       break;
 993     case T_BOOLEAN: {
 994       __ ldrb(dest->as_register(), as_Address(from_addr));
 995       break;
 996     }
 997 
 998     case T_CHAR:
 999       __ ldrh(dest->as_register(), as_Address(from_addr));
1000       break;
1001     case T_SHORT:
1002       __ ldrsh(dest->as_register(), as_Address(from_addr));
1003       break;
1004 
1005     default:
1006       ShouldNotReachHere();
1007   }
1008 
1009   if (is_reference_type(type)) {
1010     if (UseCompressedOops && !wide) {
1011       __ decode_heap_oop(dest->as_register());
1012     }
1013 
1014     if (!UseZGC) {
1015       // Load barrier has not yet been applied, so ZGC can't verify the oop here
1016       __ verify_oop(dest->as_register());
1017     }
1018   }
1019 }
1020 
1021 
1022 int LIR_Assembler::array_element_size(BasicType type) const {
1023   int elem_size = type2aelembytes(type);
1024   return exact_log2(elem_size);
1025 }
1026 
1027 
1028 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1029   switch (op->code()) {
1030   case lir_idiv:
1031   case lir_irem:
1032     arithmetic_idiv(op->code(),
1033                     op->in_opr1(),
1034                     op->in_opr2(),
1035                     op->in_opr3(),
1036                     op->result_opr(),
1037                     op->info());
1038     break;
1039   case lir_fmad:
1040     __ fmaddd(op->result_opr()->as_double_reg(),
1041               op->in_opr1()->as_double_reg(),
1042               op->in_opr2()->as_double_reg(),
1043               op->in_opr3()->as_double_reg());
1044     break;
1045   case lir_fmaf:
1046     __ fmadds(op->result_opr()->as_float_reg(),
1047               op->in_opr1()->as_float_reg(),
1048               op->in_opr2()->as_float_reg(),
1049               op->in_opr3()->as_float_reg());
1050     break;
1051   default:      ShouldNotReachHere(); break;
1052   }
1053 }
1054 
1055 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1056 #ifdef ASSERT
1057   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1058   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1059   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1060 #endif
1061 
1062   if (op->cond() == lir_cond_always) {
1063     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1064     __ b(*(op->label()));
1065   } else {
1066     Assembler::Condition acond;
1067     if (op->code() == lir_cond_float_branch) {
1068       bool is_unordered = (op->ublock() == op->block());
1069       // Assembler::EQ does not permit unordered branches, so we add
1070       // another branch here.  Likewise, Assembler::NE does not permit
1071       // ordered branches.
1072       if ((is_unordered && op->cond() == lir_cond_equal)
1073           || (!is_unordered && op->cond() == lir_cond_notEqual))
1074         __ br(Assembler::VS, *(op->ublock()->label()));
1075       switch(op->cond()) {
1076       case lir_cond_equal:        acond = Assembler::EQ; break;
1077       case lir_cond_notEqual:     acond = Assembler::NE; break;
1078       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1079       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1080       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1081       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1082       default:                    ShouldNotReachHere();
1083         acond = Assembler::EQ;  // unreachable
1084       }
1085     } else {
1086       switch (op->cond()) {
1087         case lir_cond_equal:        acond = Assembler::EQ; break;
1088         case lir_cond_notEqual:     acond = Assembler::NE; break;
1089         case lir_cond_less:         acond = Assembler::LT; break;
1090         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1091         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1092         case lir_cond_greater:      acond = Assembler::GT; break;
1093         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1094         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1095         default:                    ShouldNotReachHere();
1096           acond = Assembler::EQ;  // unreachable
1097       }
1098     }
1099     __ br(acond,*(op->label()));
1100   }
1101 }
1102 
1103 
1104 
1105 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1106   LIR_Opr src  = op->in_opr();
1107   LIR_Opr dest = op->result_opr();
1108 
1109   switch (op->bytecode()) {
1110     case Bytecodes::_i2f:
1111       {
1112         __ scvtfws(dest->as_float_reg(), src->as_register());
1113         break;
1114       }
1115     case Bytecodes::_i2d:
1116       {
1117         __ scvtfwd(dest->as_double_reg(), src->as_register());
1118         break;
1119       }
1120     case Bytecodes::_l2d:
1121       {
1122         __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1123         break;
1124       }
1125     case Bytecodes::_l2f:
1126       {
1127         __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1128         break;
1129       }
1130     case Bytecodes::_f2d:
1131       {
1132         __ fcvts(dest->as_double_reg(), src->as_float_reg());
1133         break;
1134       }
1135     case Bytecodes::_d2f:
1136       {
1137         __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1138         break;
1139       }
1140     case Bytecodes::_i2c:
1141       {
1142         __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1143         break;
1144       }
1145     case Bytecodes::_i2l:
1146       {
1147         __ sxtw(dest->as_register_lo(), src->as_register());
1148         break;
1149       }
1150     case Bytecodes::_i2s:
1151       {
1152         __ sxth(dest->as_register(), src->as_register());
1153         break;
1154       }
1155     case Bytecodes::_i2b:
1156       {
1157         __ sxtb(dest->as_register(), src->as_register());
1158         break;
1159       }
1160     case Bytecodes::_l2i:
1161       {
1162         _masm->block_comment("FIXME: This could be a no-op");
1163         __ uxtw(dest->as_register(), src->as_register_lo());
1164         break;
1165       }
1166     case Bytecodes::_d2l:
1167       {
1168         __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1169         break;
1170       }
1171     case Bytecodes::_f2i:
1172       {
1173         __ fcvtzsw(dest->as_register(), src->as_float_reg());
1174         break;
1175       }
1176     case Bytecodes::_f2l:
1177       {
1178         __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1179         break;
1180       }
1181     case Bytecodes::_d2i:
1182       {
1183         __ fcvtzdw(dest->as_register(), src->as_double_reg());
1184         break;
1185       }
1186     default: ShouldNotReachHere();
1187   }
1188 }
1189 
1190 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1191   if (op->init_check()) {
1192     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1193                                InstanceKlass::init_state_offset()));
1194     __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1195     add_debug_info_for_null_check_here(op->stub()->info());
1196     __ br(Assembler::NE, *op->stub()->entry());
1197   }
1198   __ allocate_object(op->obj()->as_register(),
1199                      op->tmp1()->as_register(),
1200                      op->tmp2()->as_register(),
1201                      op->header_size(),
1202                      op->object_size(),
1203                      op->klass()->as_register(),
1204                      *op->stub()->entry());
1205   __ bind(*op->stub()->continuation());
1206 }
1207 
1208 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1209   Register len =  op->len()->as_register();
1210   __ uxtw(len, len);
1211 
1212   if (UseSlowPath ||
1213       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1214       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1215     __ b(*op->stub()->entry());
1216   } else {
1217     Register tmp1 = op->tmp1()->as_register();
1218     Register tmp2 = op->tmp2()->as_register();
1219     Register tmp3 = op->tmp3()->as_register();
1220     if (len == tmp1) {
1221       tmp1 = tmp3;
1222     } else if (len == tmp2) {
1223       tmp2 = tmp3;
1224     } else if (len == tmp3) {
1225       // everything is ok
1226     } else {
1227       __ mov(tmp3, len);
1228     }
1229     __ allocate_array(op->obj()->as_register(),
1230                       len,
1231                       tmp1,
1232                       tmp2,
1233                       arrayOopDesc::base_offset_in_bytes(op->type()),
1234                       array_element_size(op->type()),
1235                       op->klass()->as_register(),
1236                       *op->stub()->entry());
1237   }
1238   __ bind(*op->stub()->continuation());
1239 }
1240 
1241 void LIR_Assembler::type_profile_helper(Register mdo,
1242                                         ciMethodData *md, ciProfileData *data,
1243                                         Register recv, Label* update_done) {
1244   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1245     Label next_test;
1246     // See if the receiver is receiver[n].
1247     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1248     __ ldr(rscratch1, Address(rscratch2));
1249     __ cmp(recv, rscratch1);
1250     __ br(Assembler::NE, next_test);
1251     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1252     __ addptr(data_addr, DataLayout::counter_increment);
1253     __ b(*update_done);
1254     __ bind(next_test);
1255   }
1256 
1257   // Didn't find receiver; find next empty slot and fill it in
1258   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1259     Label next_test;
1260     __ lea(rscratch2,
1261            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1262     Address recv_addr(rscratch2);
1263     __ ldr(rscratch1, recv_addr);
1264     __ cbnz(rscratch1, next_test);
1265     __ str(recv, recv_addr);
1266     __ mov(rscratch1, DataLayout::counter_increment);
1267     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1268     __ str(rscratch1, Address(rscratch2));
1269     __ b(*update_done);
1270     __ bind(next_test);
1271   }
1272 }
1273 
1274 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1275   // we always need a stub for the failure case.
1276   CodeStub* stub = op->stub();
1277   Register obj = op->object()->as_register();
1278   Register k_RInfo = op->tmp1()->as_register();
1279   Register klass_RInfo = op->tmp2()->as_register();
1280   Register dst = op->result_opr()->as_register();
1281   ciKlass* k = op->klass();
1282   Register Rtmp1 = noreg;
1283 
1284   // check if it needs to be profiled
1285   ciMethodData* md;
1286   ciProfileData* data;
1287 
1288   const bool should_profile = op->should_profile();
1289 
1290   if (should_profile) {
1291     ciMethod* method = op->profiled_method();
1292     assert(method != NULL, "Should have method");
1293     int bci = op->profiled_bci();
1294     md = method->method_data_or_null();
1295     assert(md != NULL, "Sanity");
1296     data = md->bci_to_data(bci);
1297     assert(data != NULL,                "need data for type check");
1298     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1299   }
1300   Label profile_cast_success, profile_cast_failure;
1301   Label *success_target = should_profile ? &profile_cast_success : success;
1302   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1303 
1304   if (obj == k_RInfo) {
1305     k_RInfo = dst;
1306   } else if (obj == klass_RInfo) {
1307     klass_RInfo = dst;
1308   }
1309   if (k->is_loaded() && !UseCompressedClassPointers) {
1310     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1311   } else {
1312     Rtmp1 = op->tmp3()->as_register();
1313     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1314   }
1315 
1316   assert_different_registers(obj, k_RInfo, klass_RInfo);
1317 
1318     if (should_profile) {
1319       Label not_null;
1320       __ cbnz(obj, not_null);
1321       // Object is null; update MDO and exit
1322       Register mdo  = klass_RInfo;
1323       __ mov_metadata(mdo, md->constant_encoding());
1324       Address data_addr
1325         = __ form_address(rscratch2, mdo,
1326                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1327                           0);
1328       __ ldrb(rscratch1, data_addr);
1329       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1330       __ strb(rscratch1, data_addr);
1331       __ b(*obj_is_null);
1332       __ bind(not_null);
1333     } else {
1334       __ cbz(obj, *obj_is_null);
1335     }
1336 
1337   if (!k->is_loaded()) {
1338     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1339   } else {
1340     __ mov_metadata(k_RInfo, k->constant_encoding());
1341   }
1342   __ verify_oop(obj);
1343 
1344   if (op->fast_check()) {
1345     // get object class
1346     // not a safepoint as obj null check happens earlier
1347     __ load_klass(rscratch1, obj);
1348     __ cmp( rscratch1, k_RInfo);
1349 
1350     __ br(Assembler::NE, *failure_target);
1351     // successful cast, fall through to profile or jump
1352   } else {
1353     // get object class
1354     // not a safepoint as obj null check happens earlier
1355     __ load_klass(klass_RInfo, obj);
1356     if (k->is_loaded()) {
1357       // See if we get an immediate positive hit
1358       __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1359       __ cmp(k_RInfo, rscratch1);
1360       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1361         __ br(Assembler::NE, *failure_target);
1362         // successful cast, fall through to profile or jump
1363       } else {
1364         // See if we get an immediate positive hit
1365         __ br(Assembler::EQ, *success_target);
1366         // check for self
1367         __ cmp(klass_RInfo, k_RInfo);
1368         __ br(Assembler::EQ, *success_target);
1369 
1370         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1371         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1372         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1373         // result is a boolean
1374         __ cbzw(klass_RInfo, *failure_target);
1375         // successful cast, fall through to profile or jump
1376       }
1377     } else {
1378       // perform the fast part of the checking logic
1379       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1380       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1381       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1382       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1383       __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1384       // result is a boolean
1385       __ cbz(k_RInfo, *failure_target);
1386       // successful cast, fall through to profile or jump
1387     }
1388   }
1389   if (should_profile) {
1390     Register mdo  = klass_RInfo, recv = k_RInfo;
1391     __ bind(profile_cast_success);
1392     __ mov_metadata(mdo, md->constant_encoding());
1393     __ load_klass(recv, obj);
1394     Label update_done;
1395     type_profile_helper(mdo, md, data, recv, success);
1396     __ b(*success);
1397 
1398     __ bind(profile_cast_failure);
1399     __ mov_metadata(mdo, md->constant_encoding());
1400     Address counter_addr
1401       = __ form_address(rscratch2, mdo,
1402                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1403                         0);
1404     __ ldr(rscratch1, counter_addr);
1405     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1406     __ str(rscratch1, counter_addr);
1407     __ b(*failure);
1408   }
1409   __ b(*success);
1410 }
1411 
1412 
1413 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1414   const bool should_profile = op->should_profile();
1415 
1416   LIR_Code code = op->code();
1417   if (code == lir_store_check) {
1418     Register value = op->object()->as_register();
1419     Register array = op->array()->as_register();
1420     Register k_RInfo = op->tmp1()->as_register();
1421     Register klass_RInfo = op->tmp2()->as_register();
1422     Register Rtmp1 = op->tmp3()->as_register();
1423 
1424     CodeStub* stub = op->stub();
1425 
1426     // check if it needs to be profiled
1427     ciMethodData* md;
1428     ciProfileData* data;
1429 
1430     if (should_profile) {
1431       ciMethod* method = op->profiled_method();
1432       assert(method != NULL, "Should have method");
1433       int bci = op->profiled_bci();
1434       md = method->method_data_or_null();
1435       assert(md != NULL, "Sanity");
1436       data = md->bci_to_data(bci);
1437       assert(data != NULL,                "need data for type check");
1438       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1439     }
1440     Label profile_cast_success, profile_cast_failure, done;
1441     Label *success_target = should_profile ? &profile_cast_success : &done;
1442     Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1443 
1444     if (should_profile) {
1445       Label not_null;
1446       __ cbnz(value, not_null);
1447       // Object is null; update MDO and exit
1448       Register mdo  = klass_RInfo;
1449       __ mov_metadata(mdo, md->constant_encoding());
1450       Address data_addr
1451         = __ form_address(rscratch2, mdo,
1452                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1453                           0);
1454       __ ldrb(rscratch1, data_addr);
1455       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1456       __ strb(rscratch1, data_addr);
1457       __ b(done);
1458       __ bind(not_null);
1459     } else {
1460       __ cbz(value, done);
1461     }
1462 
1463     add_debug_info_for_null_check_here(op->info_for_exception());
1464     __ load_klass(k_RInfo, array);
1465     __ load_klass(klass_RInfo, value);
1466 
1467     // get instance klass (it's already uncompressed)
1468     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1469     // perform the fast part of the checking logic
1470     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1471     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1472     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1473     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1474     __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1475     // result is a boolean
1476     __ cbzw(k_RInfo, *failure_target);
1477     // fall through to the success case
1478 
1479     if (should_profile) {
1480       Register mdo  = klass_RInfo, recv = k_RInfo;
1481       __ bind(profile_cast_success);
1482       __ mov_metadata(mdo, md->constant_encoding());
1483       __ load_klass(recv, value);
1484       Label update_done;
1485       type_profile_helper(mdo, md, data, recv, &done);
1486       __ b(done);
1487 
1488       __ bind(profile_cast_failure);
1489       __ mov_metadata(mdo, md->constant_encoding());
1490       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1491       __ lea(rscratch2, counter_addr);
1492       __ ldr(rscratch1, Address(rscratch2));
1493       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1494       __ str(rscratch1, Address(rscratch2));
1495       __ b(*stub->entry());
1496     }
1497 
1498     __ bind(done);
1499   } else if (code == lir_checkcast) {
1500     Register obj = op->object()->as_register();
1501     Register dst = op->result_opr()->as_register();
1502     Label success;
1503     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1504     __ bind(success);
1505     if (dst != obj) {
1506       __ mov(dst, obj);
1507     }
1508   } else if (code == lir_instanceof) {
1509     Register obj = op->object()->as_register();
1510     Register dst = op->result_opr()->as_register();
1511     Label success, failure, done;
1512     emit_typecheck_helper(op, &success, &failure, &failure);
1513     __ bind(failure);
1514     __ mov(dst, zr);
1515     __ b(done);
1516     __ bind(success);
1517     __ mov(dst, 1);
1518     __ bind(done);
1519   } else {
1520     ShouldNotReachHere();
1521   }
1522 }
1523 
1524 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1525   __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1526   __ cset(rscratch1, Assembler::NE);
1527   __ membar(__ AnyAny);
1528 }
1529 
1530 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1531   __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1532   __ cset(rscratch1, Assembler::NE);
1533   __ membar(__ AnyAny);
1534 }
1535 
1536 
1537 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1538   assert(VM_Version::supports_cx8(), "wrong machine");
1539   Register addr;
1540   if (op->addr()->is_register()) {
1541     addr = as_reg(op->addr());
1542   } else {
1543     assert(op->addr()->is_address(), "what else?");
1544     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1545     assert(addr_ptr->disp() == 0, "need 0 disp");
1546     assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1547     addr = as_reg(addr_ptr->base());
1548   }
1549   Register newval = as_reg(op->new_value());
1550   Register cmpval = as_reg(op->cmp_value());
1551 
1552   if (op->code() == lir_cas_obj) {
1553     if (UseCompressedOops) {
1554       Register t1 = op->tmp1()->as_register();
1555       assert(op->tmp1()->is_valid(), "must be");
1556       __ encode_heap_oop(t1, cmpval);
1557       cmpval = t1;
1558       __ encode_heap_oop(rscratch2, newval);
1559       newval = rscratch2;
1560       casw(addr, newval, cmpval);
1561     } else {
1562       casl(addr, newval, cmpval);
1563     }
1564   } else if (op->code() == lir_cas_int) {
1565     casw(addr, newval, cmpval);
1566   } else {
1567     casl(addr, newval, cmpval);
1568   }
1569 }
1570 
1571 
1572 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1573                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1574   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on aarch64");
1575 
1576   Assembler::Condition acond, ncond;
1577   switch (condition) {
1578   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1579   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1580   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1581   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1582   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1583   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1584   case lir_cond_belowEqual:
1585   case lir_cond_aboveEqual:
1586   default:                    ShouldNotReachHere();
1587     acond = Assembler::EQ; ncond = Assembler::NE;  // unreachable
1588   }
1589 
1590   assert(result->is_single_cpu() || result->is_double_cpu(),
1591          "expect single register for result");
1592   if (opr1->is_constant() && opr2->is_constant()
1593       && opr1->type() == T_INT && opr2->type() == T_INT) {
1594     jint val1 = opr1->as_jint();
1595     jint val2 = opr2->as_jint();
1596     if (val1 == 0 && val2 == 1) {
1597       __ cset(result->as_register(), ncond);
1598       return;
1599     } else if (val1 == 1 && val2 == 0) {
1600       __ cset(result->as_register(), acond);
1601       return;
1602     }
1603   }
1604 
1605   if (opr1->is_constant() && opr2->is_constant()
1606       && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1607     jlong val1 = opr1->as_jlong();
1608     jlong val2 = opr2->as_jlong();
1609     if (val1 == 0 && val2 == 1) {
1610       __ cset(result->as_register_lo(), ncond);
1611       return;
1612     } else if (val1 == 1 && val2 == 0) {
1613       __ cset(result->as_register_lo(), acond);
1614       return;
1615     }
1616   }
1617 
1618   if (opr1->is_stack()) {
1619     stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1620     opr1 = FrameMap::rscratch1_opr;
1621   } else if (opr1->is_constant()) {
1622     LIR_Opr tmp
1623       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1624     const2reg(opr1, tmp, lir_patch_none, NULL);
1625     opr1 = tmp;
1626   }
1627 
1628   if (opr2->is_stack()) {
1629     stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1630     opr2 = FrameMap::rscratch2_opr;
1631   } else if (opr2->is_constant()) {
1632     LIR_Opr tmp
1633       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1634     const2reg(opr2, tmp, lir_patch_none, NULL);
1635     opr2 = tmp;
1636   }
1637 
1638   if (result->type() == T_LONG)
1639     __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1640   else
1641     __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1642 }
1643 
1644 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1645   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1646 
1647   if (left->is_single_cpu()) {
1648     Register lreg = left->as_register();
1649     Register dreg = as_reg(dest);
1650 
1651     if (right->is_single_cpu()) {
1652       // cpu register - cpu register
1653 
1654       assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1655              "should be");
1656       Register rreg = right->as_register();
1657       switch (code) {
1658       case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1659       case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1660       case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1661       default:      ShouldNotReachHere();
1662       }
1663 
1664     } else if (right->is_double_cpu()) {
1665       Register rreg = right->as_register_lo();
1666       // single_cpu + double_cpu: can happen with obj+long
1667       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1668       switch (code) {
1669       case lir_add: __ add(dreg, lreg, rreg); break;
1670       case lir_sub: __ sub(dreg, lreg, rreg); break;
1671       default: ShouldNotReachHere();
1672       }
1673     } else if (right->is_constant()) {
1674       // cpu register - constant
1675       jlong c;
1676 
1677       // FIXME.  This is fugly: we really need to factor all this logic.
1678       switch(right->type()) {
1679       case T_LONG:
1680         c = right->as_constant_ptr()->as_jlong();
1681         break;
1682       case T_INT:
1683       case T_ADDRESS:
1684         c = right->as_constant_ptr()->as_jint();
1685         break;
1686       default:
1687         ShouldNotReachHere();
1688         c = 0;  // unreachable
1689         break;
1690       }
1691 
1692       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1693       if (c == 0 && dreg == lreg) {
1694         COMMENT("effective nop elided");
1695         return;
1696       }
1697       switch(left->type()) {
1698       case T_INT:
1699         switch (code) {
1700         case lir_add: __ addw(dreg, lreg, c); break;
1701         case lir_sub: __ subw(dreg, lreg, c); break;
1702         default: ShouldNotReachHere();
1703         }
1704         break;
1705       case T_OBJECT:
1706       case T_ADDRESS:
1707         switch (code) {
1708         case lir_add: __ add(dreg, lreg, c); break;
1709         case lir_sub: __ sub(dreg, lreg, c); break;
1710         default: ShouldNotReachHere();
1711         }
1712         break;
1713       default:
1714         ShouldNotReachHere();
1715       }
1716     } else {
1717       ShouldNotReachHere();
1718     }
1719 
1720   } else if (left->is_double_cpu()) {
1721     Register lreg_lo = left->as_register_lo();
1722 
1723     if (right->is_double_cpu()) {
1724       // cpu register - cpu register
1725       Register rreg_lo = right->as_register_lo();
1726       switch (code) {
1727       case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1728       case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1729       case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1730       case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1731       case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1732       default:
1733         ShouldNotReachHere();
1734       }
1735 
1736     } else if (right->is_constant()) {
1737       jlong c = right->as_constant_ptr()->as_jlong();
1738       Register dreg = as_reg(dest);
1739       switch (code) {
1740         case lir_add:
1741         case lir_sub:
1742           if (c == 0 && dreg == lreg_lo) {
1743             COMMENT("effective nop elided");
1744             return;
1745           }
1746           code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c);
1747           break;
1748         case lir_div:
1749           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1750           if (c == 1) {
1751             // move lreg_lo to dreg if divisor is 1
1752             __ mov(dreg, lreg_lo);
1753           } else {
1754             unsigned int shift = log2i_exact(c);
1755             // use rscratch1 as intermediate result register
1756             __ asr(rscratch1, lreg_lo, 63);
1757             __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift);
1758             __ asr(dreg, rscratch1, shift);
1759           }
1760           break;
1761         case lir_rem:
1762           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1763           if (c == 1) {
1764             // move 0 to dreg if divisor is 1
1765             __ mov(dreg, zr);
1766           } else {
1767             // use rscratch1 as intermediate result register
1768             __ negs(rscratch1, lreg_lo);
1769             __ andr(dreg, lreg_lo, c - 1);
1770             __ andr(rscratch1, rscratch1, c - 1);
1771             __ csneg(dreg, dreg, rscratch1, Assembler::MI);
1772           }
1773           break;
1774         default:
1775           ShouldNotReachHere();
1776       }
1777     } else {
1778       ShouldNotReachHere();
1779     }
1780   } else if (left->is_single_fpu()) {
1781     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1782     switch (code) {
1783     case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1784     case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1785     case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1786     case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1787     default:
1788       ShouldNotReachHere();
1789     }
1790   } else if (left->is_double_fpu()) {
1791     if (right->is_double_fpu()) {
1792       // fpu register - fpu register
1793       switch (code) {
1794       case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1795       case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1796       case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1797       case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1798       default:
1799         ShouldNotReachHere();
1800       }
1801     } else {
1802       if (right->is_constant()) {
1803         ShouldNotReachHere();
1804       }
1805       ShouldNotReachHere();
1806     }
1807   } else if (left->is_single_stack() || left->is_address()) {
1808     assert(left == dest, "left and dest must be equal");
1809     ShouldNotReachHere();
1810   } else {
1811     ShouldNotReachHere();
1812   }
1813 }
1814 
1815 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1816 
1817 
1818 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1819   switch(code) {
1820   case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1821   case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1822   default      : ShouldNotReachHere();
1823   }
1824 }
1825 
1826 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1827 
1828   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1829   Register Rleft = left->is_single_cpu() ? left->as_register() :
1830                                            left->as_register_lo();
1831    if (dst->is_single_cpu()) {
1832      Register Rdst = dst->as_register();
1833      if (right->is_constant()) {
1834        switch (code) {
1835          case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1836          case lir_logic_or:  __ orrw (Rdst, Rleft, right->as_jint()); break;
1837          case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1838          default: ShouldNotReachHere(); break;
1839        }
1840      } else {
1841        Register Rright = right->is_single_cpu() ? right->as_register() :
1842                                                   right->as_register_lo();
1843        switch (code) {
1844          case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1845          case lir_logic_or:  __ orrw (Rdst, Rleft, Rright); break;
1846          case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1847          default: ShouldNotReachHere(); break;
1848        }
1849      }
1850    } else {
1851      Register Rdst = dst->as_register_lo();
1852      if (right->is_constant()) {
1853        switch (code) {
1854          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1855          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jlong()); break;
1856          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1857          default: ShouldNotReachHere(); break;
1858        }
1859      } else {
1860        Register Rright = right->is_single_cpu() ? right->as_register() :
1861                                                   right->as_register_lo();
1862        switch (code) {
1863          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1864          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
1865          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1866          default: ShouldNotReachHere(); break;
1867        }
1868      }
1869    }
1870 }
1871 
1872 
1873 
1874 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) {
1875 
1876   // opcode check
1877   assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem");
1878   bool is_irem = (code == lir_irem);
1879 
1880   // operand check
1881   assert(left->is_single_cpu(),   "left must be register");
1882   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
1883   assert(result->is_single_cpu(), "result must be register");
1884   Register lreg = left->as_register();
1885   Register dreg = result->as_register();
1886 
1887   // power-of-2 constant check and codegen
1888   if (right->is_constant()) {
1889     int c = right->as_constant_ptr()->as_jint();
1890     assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1891     if (is_irem) {
1892       if (c == 1) {
1893         // move 0 to dreg if divisor is 1
1894         __ movw(dreg, zr);
1895       } else {
1896         // use rscratch1 as intermediate result register
1897         __ negsw(rscratch1, lreg);
1898         __ andw(dreg, lreg, c - 1);
1899         __ andw(rscratch1, rscratch1, c - 1);
1900         __ csnegw(dreg, dreg, rscratch1, Assembler::MI);
1901       }
1902     } else {
1903       if (c == 1) {
1904         // move lreg to dreg if divisor is 1
1905         __ movw(dreg, lreg);
1906       } else {
1907         unsigned int shift = exact_log2(c);
1908         // use rscratch1 as intermediate result register
1909         __ asrw(rscratch1, lreg, 31);
1910         __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift);
1911         __ asrw(dreg, rscratch1, shift);
1912       }
1913     }
1914   } else {
1915     Register rreg = right->as_register();
1916     __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1);
1917   }
1918 }
1919 
1920 
1921 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1922   if (opr1->is_constant() && opr2->is_single_cpu()) {
1923     // tableswitch
1924     Register reg = as_reg(opr2);
1925     struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1926     __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1927   } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1928     Register reg1 = as_reg(opr1);
1929     if (opr2->is_single_cpu()) {
1930       // cpu register - cpu register
1931       Register reg2 = opr2->as_register();
1932       if (is_reference_type(opr1->type())) {
1933         __ cmpoop(reg1, reg2);
1934       } else {
1935         assert(!is_reference_type(opr2->type()), "cmp int, oop?");
1936         __ cmpw(reg1, reg2);
1937       }
1938       return;
1939     }
1940     if (opr2->is_double_cpu()) {
1941       // cpu register - cpu register
1942       Register reg2 = opr2->as_register_lo();
1943       __ cmp(reg1, reg2);
1944       return;
1945     }
1946 
1947     if (opr2->is_constant()) {
1948       bool is_32bit = false; // width of register operand
1949       jlong imm;
1950 
1951       switch(opr2->type()) {
1952       case T_INT:
1953         imm = opr2->as_constant_ptr()->as_jint();
1954         is_32bit = true;
1955         break;
1956       case T_LONG:
1957         imm = opr2->as_constant_ptr()->as_jlong();
1958         break;
1959       case T_ADDRESS:
1960         imm = opr2->as_constant_ptr()->as_jint();
1961         break;
1962       case T_METADATA:
1963         imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
1964         break;
1965       case T_OBJECT:
1966       case T_ARRAY:
1967         jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
1968         __ cmpoop(reg1, rscratch1);
1969         return;
1970       default:
1971         ShouldNotReachHere();
1972         imm = 0;  // unreachable
1973         break;
1974       }
1975 
1976       if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1977         if (is_32bit)
1978           __ cmpw(reg1, imm);
1979         else
1980           __ subs(zr, reg1, imm);
1981         return;
1982       } else {
1983         __ mov(rscratch1, imm);
1984         if (is_32bit)
1985           __ cmpw(reg1, rscratch1);
1986         else
1987           __ cmp(reg1, rscratch1);
1988         return;
1989       }
1990     } else
1991       ShouldNotReachHere();
1992   } else if (opr1->is_single_fpu()) {
1993     FloatRegister reg1 = opr1->as_float_reg();
1994     assert(opr2->is_single_fpu(), "expect single float register");
1995     FloatRegister reg2 = opr2->as_float_reg();
1996     __ fcmps(reg1, reg2);
1997   } else if (opr1->is_double_fpu()) {
1998     FloatRegister reg1 = opr1->as_double_reg();
1999     assert(opr2->is_double_fpu(), "expect double float register");
2000     FloatRegister reg2 = opr2->as_double_reg();
2001     __ fcmpd(reg1, reg2);
2002   } else {
2003     ShouldNotReachHere();
2004   }
2005 }
2006 
2007 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2008   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2009     bool is_unordered_less = (code == lir_ucmp_fd2i);
2010     if (left->is_single_fpu()) {
2011       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2012     } else if (left->is_double_fpu()) {
2013       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2014     } else {
2015       ShouldNotReachHere();
2016     }
2017   } else if (code == lir_cmp_l2i) {
2018     Label done;
2019     __ cmp(left->as_register_lo(), right->as_register_lo());
2020     __ mov(dst->as_register(), (uint64_t)-1L);
2021     __ br(Assembler::LT, done);
2022     __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2023     __ bind(done);
2024   } else {
2025     ShouldNotReachHere();
2026   }
2027 }
2028 
2029 
2030 void LIR_Assembler::align_call(LIR_Code code) {  }
2031 
2032 
2033 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2034   address call = __ trampoline_call(Address(op->addr(), rtype));
2035   if (call == NULL) {
2036     bailout("trampoline stub overflow");
2037     return;
2038   }
2039   add_call_info(code_offset(), op->info());
2040   __ post_call_nop();
2041 }
2042 
2043 
2044 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2045   address call = __ ic_call(op->addr());
2046   if (call == NULL) {
2047     bailout("trampoline stub overflow");
2048     return;
2049   }
2050   add_call_info(code_offset(), op->info());
2051   __ post_call_nop();
2052 }
2053 
2054 void LIR_Assembler::emit_static_call_stub() {
2055   address call_pc = __ pc();
2056   address stub = __ start_a_stub(call_stub_size());
2057   if (stub == NULL) {
2058     bailout("static call stub overflow");
2059     return;
2060   }
2061 
2062   int start = __ offset();
2063 
2064   __ relocate(static_stub_Relocation::spec(call_pc));
2065   __ emit_static_call_stub();
2066 
2067   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
2068         <= call_stub_size(), "stub too big");
2069   __ end_a_stub();
2070 }
2071 
2072 
2073 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2074   assert(exceptionOop->as_register() == r0, "must match");
2075   assert(exceptionPC->as_register() == r3, "must match");
2076 
2077   // exception object is not added to oop map by LinearScan
2078   // (LinearScan assumes that no oops are in fixed registers)
2079   info->add_register_oop(exceptionOop);
2080   Runtime1::StubID unwind_id;
2081 
2082   // get current pc information
2083   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2084   if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
2085     // As no instructions have been generated yet for this LIR node it's
2086     // possible that an oop map already exists for the current offset.
2087     // In that case insert an dummy NOP here to ensure all oop map PCs
2088     // are unique. See JDK-8237483.
2089     __ nop();
2090   }
2091   int pc_for_athrow_offset = __ offset();
2092   InternalAddress pc_for_athrow(__ pc());
2093   __ adr(exceptionPC->as_register(), pc_for_athrow);
2094   add_call_info(pc_for_athrow_offset, info); // for exception handler
2095 
2096   __ verify_not_null_oop(r0);
2097   // search an exception handler (r0: exception oop, r3: throwing pc)
2098   if (compilation()->has_fpu_code()) {
2099     unwind_id = Runtime1::handle_exception_id;
2100   } else {
2101     unwind_id = Runtime1::handle_exception_nofpu_id;
2102   }
2103   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2104 
2105   // FIXME: enough room for two byte trap   ????
2106   __ nop();
2107 }
2108 
2109 
2110 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2111   assert(exceptionOop->as_register() == r0, "must match");
2112 
2113   __ b(_unwind_handler_entry);
2114 }
2115 
2116 
2117 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2118   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2119   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2120 
2121   switch (left->type()) {
2122     case T_INT: {
2123       switch (code) {
2124       case lir_shl:  __ lslvw (dreg, lreg, count->as_register()); break;
2125       case lir_shr:  __ asrvw (dreg, lreg, count->as_register()); break;
2126       case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2127       default:
2128         ShouldNotReachHere();
2129         break;
2130       }
2131       break;
2132     case T_LONG:
2133     case T_ADDRESS:
2134     case T_OBJECT:
2135       switch (code) {
2136       case lir_shl:  __ lslv (dreg, lreg, count->as_register()); break;
2137       case lir_shr:  __ asrv (dreg, lreg, count->as_register()); break;
2138       case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2139       default:
2140         ShouldNotReachHere();
2141         break;
2142       }
2143       break;
2144     default:
2145       ShouldNotReachHere();
2146       break;
2147     }
2148   }
2149 }
2150 
2151 
2152 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2153   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2154   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2155 
2156   switch (left->type()) {
2157     case T_INT: {
2158       switch (code) {
2159       case lir_shl:  __ lslw (dreg, lreg, count); break;
2160       case lir_shr:  __ asrw (dreg, lreg, count); break;
2161       case lir_ushr: __ lsrw (dreg, lreg, count); break;
2162       default:
2163         ShouldNotReachHere();
2164         break;
2165       }
2166       break;
2167     case T_LONG:
2168     case T_ADDRESS:
2169     case T_OBJECT:
2170       switch (code) {
2171       case lir_shl:  __ lsl (dreg, lreg, count); break;
2172       case lir_shr:  __ asr (dreg, lreg, count); break;
2173       case lir_ushr: __ lsr (dreg, lreg, count); break;
2174       default:
2175         ShouldNotReachHere();
2176         break;
2177       }
2178       break;
2179     default:
2180       ShouldNotReachHere();
2181       break;
2182     }
2183   }
2184 }
2185 
2186 
2187 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2188   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2189   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2190   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2191   __ str (r, Address(sp, offset_from_rsp_in_bytes));
2192 }
2193 
2194 
2195 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2196   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2197   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2198   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2199   __ mov (rscratch1, c);
2200   __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2201 }
2202 
2203 
2204 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2205   ShouldNotReachHere();
2206   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2207   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2208   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2209   __ lea(rscratch1, __ constant_oop_address(o));
2210   __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2211 }
2212 
2213 
2214 // This code replaces a call to arraycopy; no exception may
2215 // be thrown in this code, they must be thrown in the System.arraycopy
2216 // activation frame; we could save some checks if this would not be the case
2217 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2218   ciArrayKlass* default_type = op->expected_type();
2219   Register src = op->src()->as_register();
2220   Register dst = op->dst()->as_register();
2221   Register src_pos = op->src_pos()->as_register();
2222   Register dst_pos = op->dst_pos()->as_register();
2223   Register length  = op->length()->as_register();
2224   Register tmp = op->tmp()->as_register();
2225 
2226   CodeStub* stub = op->stub();
2227   int flags = op->flags();
2228   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2229   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2230 
2231   // if we don't know anything, just go through the generic arraycopy
2232   if (default_type == NULL // || basic_type == T_OBJECT
2233       ) {
2234     Label done;
2235     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2236 
2237     // Save the arguments in case the generic arraycopy fails and we
2238     // have to fall back to the JNI stub
2239     __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2240     __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2241     __ str(src,              Address(sp, 4*BytesPerWord));
2242 
2243     address copyfunc_addr = StubRoutines::generic_arraycopy();
2244     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2245 
2246     // The arguments are in java calling convention so we shift them
2247     // to C convention
2248     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2249     __ mov(c_rarg0, j_rarg0);
2250     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2251     __ mov(c_rarg1, j_rarg1);
2252     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2253     __ mov(c_rarg2, j_rarg2);
2254     assert_different_registers(c_rarg3, j_rarg4);
2255     __ mov(c_rarg3, j_rarg3);
2256     __ mov(c_rarg4, j_rarg4);
2257 #ifndef PRODUCT
2258     if (PrintC1Statistics) {
2259       __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2260     }
2261 #endif
2262     __ far_call(RuntimeAddress(copyfunc_addr));
2263 
2264     __ cbz(r0, *stub->continuation());
2265 
2266     // Reload values from the stack so they are where the stub
2267     // expects them.
2268     __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2269     __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2270     __ ldr(src,              Address(sp, 4*BytesPerWord));
2271 
2272     // r0 is -1^K where K == partial copied count
2273     __ eonw(rscratch1, r0, zr);
2274     // adjust length down and src/end pos up by partial copied count
2275     __ subw(length, length, rscratch1);
2276     __ addw(src_pos, src_pos, rscratch1);
2277     __ addw(dst_pos, dst_pos, rscratch1);
2278     __ b(*stub->entry());
2279 
2280     __ bind(*stub->continuation());
2281     return;
2282   }
2283 
2284   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2285 
2286   int elem_size = type2aelembytes(basic_type);
2287   int scale = exact_log2(elem_size);
2288 
2289   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2290   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2291   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2292   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2293 
2294   // test for NULL
2295   if (flags & LIR_OpArrayCopy::src_null_check) {
2296     __ cbz(src, *stub->entry());
2297   }
2298   if (flags & LIR_OpArrayCopy::dst_null_check) {
2299     __ cbz(dst, *stub->entry());
2300   }
2301 
2302   // If the compiler was not able to prove that exact type of the source or the destination
2303   // of the arraycopy is an array type, check at runtime if the source or the destination is
2304   // an instance type.
2305   if (flags & LIR_OpArrayCopy::type_check) {
2306     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2307       __ load_klass(tmp, dst);
2308       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2309       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2310       __ br(Assembler::GE, *stub->entry());
2311     }
2312 
2313     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2314       __ load_klass(tmp, src);
2315       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2316       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2317       __ br(Assembler::GE, *stub->entry());
2318     }
2319   }
2320 
2321   // check if negative
2322   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2323     __ cmpw(src_pos, 0);
2324     __ br(Assembler::LT, *stub->entry());
2325   }
2326   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2327     __ cmpw(dst_pos, 0);
2328     __ br(Assembler::LT, *stub->entry());
2329   }
2330 
2331   if (flags & LIR_OpArrayCopy::length_positive_check) {
2332     __ cmpw(length, 0);
2333     __ br(Assembler::LT, *stub->entry());
2334   }
2335 
2336   if (flags & LIR_OpArrayCopy::src_range_check) {
2337     __ addw(tmp, src_pos, length);
2338     __ ldrw(rscratch1, src_length_addr);
2339     __ cmpw(tmp, rscratch1);
2340     __ br(Assembler::HI, *stub->entry());
2341   }
2342   if (flags & LIR_OpArrayCopy::dst_range_check) {
2343     __ addw(tmp, dst_pos, length);
2344     __ ldrw(rscratch1, dst_length_addr);
2345     __ cmpw(tmp, rscratch1);
2346     __ br(Assembler::HI, *stub->entry());
2347   }
2348 
2349   if (flags & LIR_OpArrayCopy::type_check) {
2350     // We don't know the array types are compatible
2351     if (basic_type != T_OBJECT) {
2352       // Simple test for basic type arrays
2353       if (UseCompressedClassPointers) {
2354         __ load_nklass(tmp, src);
2355         __ load_nklass(rscratch1, dst);
2356         __ cmpw(tmp, rscratch1);
2357       } else {
2358         __ ldr(tmp, Address(src, oopDesc::klass_offset_in_bytes()));
2359         __ ldr(rscratch1, Address(dst, oopDesc::klass_offset_in_bytes()));
2360         __ cmp(tmp, rscratch1);
2361       }
2362       __ br(Assembler::NE, *stub->entry());
2363     } else {
2364       // For object arrays, if src is a sub class of dst then we can
2365       // safely do the copy.
2366       Label cont, slow;
2367 
2368 #define PUSH(r1, r2)                                    \
2369       stp(r1, r2, __ pre(sp, -2 * wordSize));
2370 
2371 #define POP(r1, r2)                                     \
2372       ldp(r1, r2, __ post(sp, 2 * wordSize));
2373 
2374       __ PUSH(src, dst);
2375 
2376       __ load_klass(src, src);
2377       __ load_klass(dst, dst);
2378 
2379       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2380 
2381       __ PUSH(src, dst);
2382       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2383       __ POP(src, dst);
2384 
2385       __ cbnz(src, cont);
2386 
2387       __ bind(slow);
2388       __ POP(src, dst);
2389 
2390       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2391       if (copyfunc_addr != NULL) { // use stub if available
2392         // src is not a sub class of dst so we have to do a
2393         // per-element check.
2394 
2395         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2396         if ((flags & mask) != mask) {
2397           // Check that at least both of them object arrays.
2398           assert(flags & mask, "one of the two should be known to be an object array");
2399 
2400           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2401             __ load_klass(tmp, src);
2402           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2403             __ load_klass(tmp, dst);
2404           }
2405           int lh_offset = in_bytes(Klass::layout_helper_offset());
2406           Address klass_lh_addr(tmp, lh_offset);
2407           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2408           __ ldrw(rscratch1, klass_lh_addr);
2409           __ mov(rscratch2, objArray_lh);
2410           __ eorw(rscratch1, rscratch1, rscratch2);
2411           __ cbnzw(rscratch1, *stub->entry());
2412         }
2413 
2414        // Spill because stubs can use any register they like and it's
2415        // easier to restore just those that we care about.
2416         __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2417         __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2418         __ str(src,              Address(sp, 4*BytesPerWord));
2419 
2420         __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2421         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2422         assert_different_registers(c_rarg0, dst, dst_pos, length);
2423         __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2424         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2425         assert_different_registers(c_rarg1, dst, length);
2426         __ uxtw(c_rarg2, length);
2427         assert_different_registers(c_rarg2, dst);
2428 
2429         __ load_klass(c_rarg4, dst);
2430         __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2431         __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2432         __ far_call(RuntimeAddress(copyfunc_addr));
2433 
2434 #ifndef PRODUCT
2435         if (PrintC1Statistics) {
2436           Label failed;
2437           __ cbnz(r0, failed);
2438           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2439           __ bind(failed);
2440         }
2441 #endif
2442 
2443         __ cbz(r0, *stub->continuation());
2444 
2445 #ifndef PRODUCT
2446         if (PrintC1Statistics) {
2447           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2448         }
2449 #endif
2450         assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2451 
2452         // Restore previously spilled arguments
2453         __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2454         __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2455         __ ldr(src,              Address(sp, 4*BytesPerWord));
2456 
2457         // return value is -1^K where K is partial copied count
2458         __ eonw(rscratch1, r0, zr);
2459         // adjust length down and src/end pos up by partial copied count
2460         __ subw(length, length, rscratch1);
2461         __ addw(src_pos, src_pos, rscratch1);
2462         __ addw(dst_pos, dst_pos, rscratch1);
2463       }
2464 
2465       __ b(*stub->entry());
2466 
2467       __ bind(cont);
2468       __ POP(src, dst);
2469     }
2470   }
2471 
2472 #ifdef ASSERT
2473   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2474     // Sanity check the known type with the incoming class.  For the
2475     // primitive case the types must match exactly with src.klass and
2476     // dst.klass each exactly matching the default type.  For the
2477     // object array case, if no type check is needed then either the
2478     // dst type is exactly the expected type and the src type is a
2479     // subtype which we can't check or src is the same array as dst
2480     // but not necessarily exactly of type default_type.
2481     Label known_ok, halt;
2482     __ mov_metadata(tmp, default_type->constant_encoding());
2483 
2484     if (basic_type != T_OBJECT) {
2485       __ cmp_klass(dst, tmp, rscratch1);
2486       __ br(Assembler::NE, halt);
2487       __ cmp_klass(src, tmp, rscratch1);
2488       __ br(Assembler::EQ, known_ok);
2489     } else {
2490       __ cmp_klass(dst, tmp, rscratch1);
2491       __ br(Assembler::EQ, known_ok);
2492       __ cmp(src, dst);
2493       __ br(Assembler::EQ, known_ok);
2494     }
2495     __ bind(halt);
2496     __ stop("incorrect type information in arraycopy");
2497     __ bind(known_ok);
2498   }
2499 #endif
2500 
2501 #ifndef PRODUCT
2502   if (PrintC1Statistics) {
2503     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2504   }
2505 #endif
2506 
2507   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2508   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2509   assert_different_registers(c_rarg0, dst, dst_pos, length);
2510   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2511   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2512   assert_different_registers(c_rarg1, dst, length);
2513   __ uxtw(c_rarg2, length);
2514   assert_different_registers(c_rarg2, dst);
2515 
2516   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2517   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2518   const char *name;
2519   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2520 
2521  CodeBlob *cb = CodeCache::find_blob(entry);
2522  if (cb) {
2523    __ far_call(RuntimeAddress(entry));
2524  } else {
2525    __ call_VM_leaf(entry, 3);
2526  }
2527 
2528   __ bind(*stub->continuation());
2529 }
2530 
2531 
2532 
2533 
2534 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2535   Register obj = op->obj_opr()->as_register();  // may not be an oop
2536   Register hdr = op->hdr_opr()->as_register();
2537   Register lock = op->lock_opr()->as_register();
2538   if (UseHeavyMonitors) {
2539     if (op->info() != NULL) {
2540       add_debug_info_for_null_check_here(op->info());
2541       __ null_check(obj, -1);
2542     }
2543     __ b(*op->stub()->entry());
2544   } else if (op->code() == lir_lock) {
2545     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2546     // add debug info for NullPointerException only if one is possible
2547     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
2548     if (op->info() != NULL) {
2549       add_debug_info_for_null_check(null_check_offset, op->info());
2550     }
2551     // done
2552   } else if (op->code() == lir_unlock) {
2553     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2554     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2555   } else {
2556     Unimplemented();
2557   }
2558   __ bind(*op->stub()->continuation());
2559 }
2560 
2561 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2562   Register obj = op->obj()->as_pointer_register();
2563   Register result = op->result_opr()->as_pointer_register();
2564 
2565   CodeEmitInfo* info = op->info();
2566   if (info != NULL) {
2567     add_debug_info_for_null_check_here(info);
2568   }
2569 
2570   if (UseCompressedClassPointers) {
2571     if (UseCompactObjectHeaders) {
2572       // Check if we can take the (common) fast path, if obj is unlocked.
2573       __ ldr(result, Address(obj, oopDesc::mark_offset_in_bytes()));
2574       __ tst(result, markWord::monitor_value);
2575       __ br(Assembler::NE, *op->stub()->entry());
2576       __ bind(*op->stub()->continuation());
2577 
2578       // Shift to get proper narrow Klass*.
2579       __ lsr(result, result, markWord::klass_shift);
2580     } else {
2581       __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2582     }
2583     __ decode_klass_not_null(result);
2584   } else {
2585     __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2586   }
2587 }
2588 
2589 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2590   ciMethod* method = op->profiled_method();
2591   int bci          = op->profiled_bci();
2592   ciMethod* callee = op->profiled_callee();
2593 
2594   // Update counter for all call types
2595   ciMethodData* md = method->method_data_or_null();
2596   assert(md != NULL, "Sanity");
2597   ciProfileData* data = md->bci_to_data(bci);
2598   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2599   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2600   Register mdo  = op->mdo()->as_register();
2601   __ mov_metadata(mdo, md->constant_encoding());
2602   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2603   // Perform additional virtual call profiling for invokevirtual and
2604   // invokeinterface bytecodes
2605   if (op->should_profile_receiver_type()) {
2606     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2607     Register recv = op->recv()->as_register();
2608     assert_different_registers(mdo, recv);
2609     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2610     ciKlass* known_klass = op->known_holder();
2611     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2612       // We know the type that will be seen at this call site; we can
2613       // statically update the MethodData* rather than needing to do
2614       // dynamic tests on the receiver type
2615 
2616       // NOTE: we should probably put a lock around this search to
2617       // avoid collisions by concurrent compilations
2618       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2619       uint i;
2620       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2621         ciKlass* receiver = vc_data->receiver(i);
2622         if (known_klass->equals(receiver)) {
2623           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2624           __ addptr(data_addr, DataLayout::counter_increment);
2625           return;
2626         }
2627       }
2628 
2629       // Receiver type not found in profile data; select an empty slot
2630 
2631       // Note that this is less efficient than it should be because it
2632       // always does a write to the receiver part of the
2633       // VirtualCallData rather than just the first time
2634       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2635         ciKlass* receiver = vc_data->receiver(i);
2636         if (receiver == NULL) {
2637           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2638           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2639           __ lea(rscratch2, recv_addr);
2640           __ str(rscratch1, Address(rscratch2));
2641           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2642           __ addptr(data_addr, DataLayout::counter_increment);
2643           return;
2644         }
2645       }
2646     } else {
2647       __ load_klass(recv, recv);
2648       Label update_done;
2649       type_profile_helper(mdo, md, data, recv, &update_done);
2650       // Receiver did not match any saved receiver and there is no empty row for it.
2651       // Increment total counter to indicate polymorphic case.
2652       __ addptr(counter_addr, DataLayout::counter_increment);
2653 
2654       __ bind(update_done);
2655     }
2656   } else {
2657     // Static call
2658     __ addptr(counter_addr, DataLayout::counter_increment);
2659   }
2660 }
2661 
2662 
2663 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2664   Unimplemented();
2665 }
2666 
2667 
2668 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2669   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2670 }
2671 
2672 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2673   assert(op->crc()->is_single_cpu(),  "crc must be register");
2674   assert(op->val()->is_single_cpu(),  "byte value must be register");
2675   assert(op->result_opr()->is_single_cpu(), "result must be register");
2676   Register crc = op->crc()->as_register();
2677   Register val = op->val()->as_register();
2678   Register res = op->result_opr()->as_register();
2679 
2680   assert_different_registers(val, crc, res);
2681   uint64_t offset;
2682   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2683   __ add(res, res, offset);
2684 
2685   __ mvnw(crc, crc); // ~crc
2686   __ update_byte_crc32(crc, val, res);
2687   __ mvnw(res, crc); // ~crc
2688 }
2689 
2690 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2691   COMMENT("emit_profile_type {");
2692   Register obj = op->obj()->as_register();
2693   Register tmp = op->tmp()->as_pointer_register();
2694   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2695   ciKlass* exact_klass = op->exact_klass();
2696   intptr_t current_klass = op->current_klass();
2697   bool not_null = op->not_null();
2698   bool no_conflict = op->no_conflict();
2699 
2700   Label update, next, none;
2701 
2702   bool do_null = !not_null;
2703   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2704   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2705 
2706   assert(do_null || do_update, "why are we here?");
2707   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2708   assert(mdo_addr.base() != rscratch1, "wrong register");
2709 
2710   __ verify_oop(obj);
2711 
2712   if (tmp != obj) {
2713     __ mov(tmp, obj);
2714   }
2715   if (do_null) {
2716     __ cbnz(tmp, update);
2717     if (!TypeEntries::was_null_seen(current_klass)) {
2718       __ ldr(rscratch2, mdo_addr);
2719       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2720       __ str(rscratch2, mdo_addr);
2721     }
2722     if (do_update) {
2723 #ifndef ASSERT
2724       __ b(next);
2725     }
2726 #else
2727       __ b(next);
2728     }
2729   } else {
2730     __ cbnz(tmp, update);
2731     __ stop("unexpected null obj");
2732 #endif
2733   }
2734 
2735   __ bind(update);
2736 
2737   if (do_update) {
2738 #ifdef ASSERT
2739     if (exact_klass != NULL) {
2740       Label ok;
2741       __ load_klass(tmp, tmp);
2742       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2743       __ eor(rscratch1, tmp, rscratch1);
2744       __ cbz(rscratch1, ok);
2745       __ stop("exact klass and actual klass differ");
2746       __ bind(ok);
2747     }
2748 #endif
2749     if (!no_conflict) {
2750       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2751         if (exact_klass != NULL) {
2752           __ mov_metadata(tmp, exact_klass->constant_encoding());
2753         } else {
2754           __ load_klass(tmp, tmp);
2755         }
2756 
2757         __ ldr(rscratch2, mdo_addr);
2758         __ eor(tmp, tmp, rscratch2);
2759         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2760         // klass seen before, nothing to do. The unknown bit may have been
2761         // set already but no need to check.
2762         __ cbz(rscratch1, next);
2763 
2764         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2765 
2766         if (TypeEntries::is_type_none(current_klass)) {
2767           __ cbz(rscratch2, none);
2768           __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2769           __ br(Assembler::EQ, none);
2770           // There is a chance that the checks above (re-reading profiling
2771           // data from memory) fail if another thread has just set the
2772           // profiling to this obj's klass
2773           __ dmb(Assembler::ISHLD);
2774           __ ldr(rscratch2, mdo_addr);
2775           __ eor(tmp, tmp, rscratch2);
2776           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2777           __ cbz(rscratch1, next);
2778         }
2779       } else {
2780         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2781                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2782 
2783         __ ldr(tmp, mdo_addr);
2784         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2785       }
2786 
2787       // different than before. Cannot keep accurate profile.
2788       __ ldr(rscratch2, mdo_addr);
2789       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2790       __ str(rscratch2, mdo_addr);
2791 
2792       if (TypeEntries::is_type_none(current_klass)) {
2793         __ b(next);
2794 
2795         __ bind(none);
2796         // first time here. Set profile type.
2797         __ str(tmp, mdo_addr);
2798       }
2799     } else {
2800       // There's a single possible klass at this profile point
2801       assert(exact_klass != NULL, "should be");
2802       if (TypeEntries::is_type_none(current_klass)) {
2803         __ mov_metadata(tmp, exact_klass->constant_encoding());
2804         __ ldr(rscratch2, mdo_addr);
2805         __ eor(tmp, tmp, rscratch2);
2806         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2807         __ cbz(rscratch1, next);
2808 #ifdef ASSERT
2809         {
2810           Label ok;
2811           __ ldr(rscratch1, mdo_addr);
2812           __ cbz(rscratch1, ok);
2813           __ cmp(rscratch1, (u1)TypeEntries::null_seen);
2814           __ br(Assembler::EQ, ok);
2815           // may have been set by another thread
2816           __ dmb(Assembler::ISHLD);
2817           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2818           __ ldr(rscratch2, mdo_addr);
2819           __ eor(rscratch2, rscratch1, rscratch2);
2820           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2821           __ cbz(rscratch2, ok);
2822 
2823           __ stop("unexpected profiling mismatch");
2824           __ bind(ok);
2825         }
2826 #endif
2827         // first time here. Set profile type.
2828         __ str(tmp, mdo_addr);
2829       } else {
2830         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2831                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2832 
2833         __ ldr(tmp, mdo_addr);
2834         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2835 
2836         __ orr(tmp, tmp, TypeEntries::type_unknown);
2837         __ str(tmp, mdo_addr);
2838         // FIXME: Write barrier needed here?
2839       }
2840     }
2841 
2842     __ bind(next);
2843   }
2844   COMMENT("} emit_profile_type");
2845 }
2846 
2847 
2848 void LIR_Assembler::align_backward_branch_target() {
2849 }
2850 
2851 
2852 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2853   // tmp must be unused
2854   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2855 
2856   if (left->is_single_cpu()) {
2857     assert(dest->is_single_cpu(), "expect single result reg");
2858     __ negw(dest->as_register(), left->as_register());
2859   } else if (left->is_double_cpu()) {
2860     assert(dest->is_double_cpu(), "expect double result reg");
2861     __ neg(dest->as_register_lo(), left->as_register_lo());
2862   } else if (left->is_single_fpu()) {
2863     assert(dest->is_single_fpu(), "expect single float result reg");
2864     __ fnegs(dest->as_float_reg(), left->as_float_reg());
2865   } else {
2866     assert(left->is_double_fpu(), "expect double float operand reg");
2867     assert(dest->is_double_fpu(), "expect double float result reg");
2868     __ fnegd(dest->as_double_reg(), left->as_double_reg());
2869   }
2870 }
2871 
2872 
2873 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2874   if (patch_code != lir_patch_none) {
2875     deoptimize_trap(info);
2876     return;
2877   }
2878 
2879   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2880 }
2881 
2882 
2883 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2884   assert(!tmp->is_valid(), "don't need temporary");
2885 
2886   CodeBlob *cb = CodeCache::find_blob(dest);
2887   if (cb) {
2888     __ far_call(RuntimeAddress(dest));
2889   } else {
2890     __ mov(rscratch1, RuntimeAddress(dest));
2891     __ blr(rscratch1);
2892   }
2893 
2894   if (info != NULL) {
2895     add_call_info_here(info);
2896   }
2897   __ post_call_nop();
2898 }
2899 
2900 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2901   if (dest->is_address() || src->is_address()) {
2902     move_op(src, dest, type, lir_patch_none, info,
2903             /*pop_fpu_stack*/false, /*wide*/false);
2904   } else {
2905     ShouldNotReachHere();
2906   }
2907 }
2908 
2909 #ifdef ASSERT
2910 // emit run-time assertion
2911 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2912   assert(op->code() == lir_assert, "must be");
2913 
2914   if (op->in_opr1()->is_valid()) {
2915     assert(op->in_opr2()->is_valid(), "both operands must be valid");
2916     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2917   } else {
2918     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2919     assert(op->condition() == lir_cond_always, "no other conditions allowed");
2920   }
2921 
2922   Label ok;
2923   if (op->condition() != lir_cond_always) {
2924     Assembler::Condition acond = Assembler::AL;
2925     switch (op->condition()) {
2926       case lir_cond_equal:        acond = Assembler::EQ;  break;
2927       case lir_cond_notEqual:     acond = Assembler::NE;  break;
2928       case lir_cond_less:         acond = Assembler::LT;  break;
2929       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
2930       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
2931       case lir_cond_greater:      acond = Assembler::GT;  break;
2932       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
2933       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
2934       default:                    ShouldNotReachHere();
2935     }
2936     __ br(acond, ok);
2937   }
2938   if (op->halt()) {
2939     const char* str = __ code_string(op->msg());
2940     __ stop(str);
2941   } else {
2942     breakpoint();
2943   }
2944   __ bind(ok);
2945 }
2946 #endif
2947 
2948 #ifndef PRODUCT
2949 #define COMMENT(x)   do { __ block_comment(x); } while (0)
2950 #else
2951 #define COMMENT(x)
2952 #endif
2953 
2954 void LIR_Assembler::membar() {
2955   COMMENT("membar");
2956   __ membar(MacroAssembler::AnyAny);
2957 }
2958 
2959 void LIR_Assembler::membar_acquire() {
2960   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2961 }
2962 
2963 void LIR_Assembler::membar_release() {
2964   __ membar(Assembler::LoadStore|Assembler::StoreStore);
2965 }
2966 
2967 void LIR_Assembler::membar_loadload() {
2968   __ membar(Assembler::LoadLoad);
2969 }
2970 
2971 void LIR_Assembler::membar_storestore() {
2972   __ membar(MacroAssembler::StoreStore);
2973 }
2974 
2975 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2976 
2977 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2978 
2979 void LIR_Assembler::on_spin_wait() {
2980   __ spin_wait();
2981 }
2982 
2983 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2984   __ mov(result_reg->as_register(), rthread);
2985 }
2986 
2987 
2988 void LIR_Assembler::peephole(LIR_List *lir) {
2989 #if 0
2990   if (tableswitch_count >= max_tableswitches)
2991     return;
2992 
2993   /*
2994     This finite-state automaton recognizes sequences of compare-and-
2995     branch instructions.  We will turn them into a tableswitch.  You
2996     could argue that C1 really shouldn't be doing this sort of
2997     optimization, but without it the code is really horrible.
2998   */
2999 
3000   enum { start_s, cmp1_s, beq_s, cmp_s } state;
3001   int first_key, last_key = -2147483648;
3002   int next_key = 0;
3003   int start_insn = -1;
3004   int last_insn = -1;
3005   Register reg = noreg;
3006   LIR_Opr reg_opr;
3007   state = start_s;
3008 
3009   LIR_OpList* inst = lir->instructions_list();
3010   for (int i = 0; i < inst->length(); i++) {
3011     LIR_Op* op = inst->at(i);
3012     switch (state) {
3013     case start_s:
3014       first_key = -1;
3015       start_insn = i;
3016       switch (op->code()) {
3017       case lir_cmp:
3018         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3019         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3020         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3021             && opr2->is_constant()
3022             && opr2->type() == T_INT) {
3023           reg_opr = opr1;
3024           reg = opr1->as_register();
3025           first_key = opr2->as_constant_ptr()->as_jint();
3026           next_key = first_key + 1;
3027           state = cmp_s;
3028           goto next_state;
3029         }
3030         break;
3031       }
3032       break;
3033     case cmp_s:
3034       switch (op->code()) {
3035       case lir_branch:
3036         if (op->as_OpBranch()->cond() == lir_cond_equal) {
3037           state = beq_s;
3038           last_insn = i;
3039           goto next_state;
3040         }
3041       }
3042       state = start_s;
3043       break;
3044     case beq_s:
3045       switch (op->code()) {
3046       case lir_cmp: {
3047         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3048         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3049         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3050             && opr1->as_register() == reg
3051             && opr2->is_constant()
3052             && opr2->type() == T_INT
3053             && opr2->as_constant_ptr()->as_jint() == next_key) {
3054           last_key = next_key;
3055           next_key++;
3056           state = cmp_s;
3057           goto next_state;
3058         }
3059       }
3060       }
3061       last_key = next_key;
3062       state = start_s;
3063       break;
3064     default:
3065       assert(false, "impossible state");
3066     }
3067     if (state == start_s) {
3068       if (first_key < last_key - 5L && reg != noreg) {
3069         {
3070           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3071           //        reg->encoding(),
3072           //        start_insn, first_key, last_key);
3073           //   for (int i = 0; i < inst->length(); i++) {
3074           //     inst->at(i)->print();
3075           //     tty->print("\n");
3076           //   }
3077           //   tty->print("\n");
3078         }
3079 
3080         struct tableswitch *sw = &switches[tableswitch_count];
3081         sw->_insn_index = start_insn, sw->_first_key = first_key,
3082           sw->_last_key = last_key, sw->_reg = reg;
3083         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3084         {
3085           // Insert the new table of branches
3086           int offset = last_insn;
3087           for (int n = first_key; n < last_key; n++) {
3088             inst->insert_before
3089               (last_insn + 1,
3090                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3091                                 inst->at(offset)->as_OpBranch()->label()));
3092             offset -= 2, i++;
3093           }
3094         }
3095         // Delete all the old compare-and-branch instructions
3096         for (int n = first_key; n < last_key; n++) {
3097           inst->remove_at(start_insn);
3098           inst->remove_at(start_insn);
3099         }
3100         // Insert the tableswitch instruction
3101         inst->insert_before(start_insn,
3102                             new LIR_Op2(lir_cmp, lir_cond_always,
3103                                         LIR_OprFact::intConst(tableswitch_count),
3104                                         reg_opr));
3105         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3106         tableswitch_count++;
3107       }
3108       reg = noreg;
3109       last_key = -2147483648;
3110     }
3111   next_state:
3112     ;
3113   }
3114 #endif
3115 }
3116 
3117 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3118   Address addr = as_Address(src->as_address_ptr());
3119   BasicType type = src->type();
3120   bool is_oop = is_reference_type(type);
3121 
3122   void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3123   void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3124 
3125   switch(type) {
3126   case T_INT:
3127     xchg = &MacroAssembler::atomic_xchgalw;
3128     add = &MacroAssembler::atomic_addalw;
3129     break;
3130   case T_LONG:
3131     xchg = &MacroAssembler::atomic_xchgal;
3132     add = &MacroAssembler::atomic_addal;
3133     break;
3134   case T_OBJECT:
3135   case T_ARRAY:
3136     if (UseCompressedOops) {
3137       xchg = &MacroAssembler::atomic_xchgalw;
3138       add = &MacroAssembler::atomic_addalw;
3139     } else {
3140       xchg = &MacroAssembler::atomic_xchgal;
3141       add = &MacroAssembler::atomic_addal;
3142     }
3143     break;
3144   default:
3145     ShouldNotReachHere();
3146     xchg = &MacroAssembler::atomic_xchgal;
3147     add = &MacroAssembler::atomic_addal; // unreachable
3148   }
3149 
3150   switch (code) {
3151   case lir_xadd:
3152     {
3153       RegisterOrConstant inc;
3154       Register tmp = as_reg(tmp_op);
3155       Register dst = as_reg(dest);
3156       if (data->is_constant()) {
3157         inc = RegisterOrConstant(as_long(data));
3158         assert_different_registers(dst, addr.base(), tmp,
3159                                    rscratch1, rscratch2);
3160       } else {
3161         inc = RegisterOrConstant(as_reg(data));
3162         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3163                                    rscratch1, rscratch2);
3164       }
3165       __ lea(tmp, addr);
3166       (_masm->*add)(dst, inc, tmp);
3167       break;
3168     }
3169   case lir_xchg:
3170     {
3171       Register tmp = tmp_op->as_register();
3172       Register obj = as_reg(data);
3173       Register dst = as_reg(dest);
3174       if (is_oop && UseCompressedOops) {
3175         __ encode_heap_oop(rscratch2, obj);
3176         obj = rscratch2;
3177       }
3178       assert_different_registers(obj, addr.base(), tmp, rscratch1);
3179       assert_different_registers(dst, addr.base(), tmp, rscratch1);
3180       __ lea(tmp, addr);
3181       (_masm->*xchg)(dst, obj, tmp);
3182       if (is_oop && UseCompressedOops) {
3183         __ decode_heap_oop(dst);
3184       }
3185     }
3186     break;
3187   default:
3188     ShouldNotReachHere();
3189   }
3190   __ membar(__ AnyAny);
3191 }
3192 
3193 #undef __