1 /*
   2  * Copyright (c) 2000, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Compilation.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_MacroAssembler.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArrayKlass.hpp"
  36 #include "ci/ciInstance.hpp"
  37 #include "code/compiledIC.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "gc/shared/gc_globals.hpp"
  40 #include "nativeInst_aarch64.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "runtime/frame.inline.hpp"
  43 #include "runtime/sharedRuntime.hpp"
  44 #include "runtime/stubRoutines.hpp"
  45 #include "utilities/powerOfTwo.hpp"
  46 #include "vmreg_aarch64.inline.hpp"
  47 
  48 
  49 #ifndef PRODUCT
  50 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  51 #else
  52 #define COMMENT(x)
  53 #endif
  54 
  55 NEEDS_CLEANUP // remove this definitions ?
  56 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  57 const Register SYNC_header = r0;   // synchronization header
  58 const Register SHIFT_count = r0;   // where count for shift operations must be
  59 
  60 #define __ _masm->
  61 
  62 
  63 static void select_different_registers(Register preserve,
  64                                        Register extra,
  65                                        Register &tmp1,
  66                                        Register &tmp2) {
  67   if (tmp1 == preserve) {
  68     assert_different_registers(tmp1, tmp2, extra);
  69     tmp1 = extra;
  70   } else if (tmp2 == preserve) {
  71     assert_different_registers(tmp1, tmp2, extra);
  72     tmp2 = extra;
  73   }
  74   assert_different_registers(preserve, tmp1, tmp2);
  75 }
  76 
  77 
  78 
  79 static void select_different_registers(Register preserve,
  80                                        Register extra,
  81                                        Register &tmp1,
  82                                        Register &tmp2,
  83                                        Register &tmp3) {
  84   if (tmp1 == preserve) {
  85     assert_different_registers(tmp1, tmp2, tmp3, extra);
  86     tmp1 = extra;
  87   } else if (tmp2 == preserve) {
  88     assert_different_registers(tmp1, tmp2, tmp3, extra);
  89     tmp2 = extra;
  90   } else if (tmp3 == preserve) {
  91     assert_different_registers(tmp1, tmp2, tmp3, extra);
  92     tmp3 = extra;
  93   }
  94   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  95 }
  96 
  97 
  98 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
  99 
 100 
 101 LIR_Opr LIR_Assembler::receiverOpr() {
 102   return FrameMap::receiver_opr;
 103 }
 104 
 105 LIR_Opr LIR_Assembler::osrBufferPointer() {
 106   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 107 }
 108 
 109 //--------------fpu register translations-----------------------
 110 
 111 
 112 address LIR_Assembler::float_constant(float f) {
 113   address const_addr = __ float_constant(f);
 114   if (const_addr == NULL) {
 115     bailout("const section overflow");
 116     return __ code()->consts()->start();
 117   } else {
 118     return const_addr;
 119   }
 120 }
 121 
 122 
 123 address LIR_Assembler::double_constant(double d) {
 124   address const_addr = __ double_constant(d);
 125   if (const_addr == NULL) {
 126     bailout("const section overflow");
 127     return __ code()->consts()->start();
 128   } else {
 129     return const_addr;
 130   }
 131 }
 132 
 133 address LIR_Assembler::int_constant(jlong n) {
 134   address const_addr = __ long_constant(n);
 135   if (const_addr == NULL) {
 136     bailout("const section overflow");
 137     return __ code()->consts()->start();
 138   } else {
 139     return const_addr;
 140   }
 141 }
 142 
 143 void LIR_Assembler::breakpoint() { Unimplemented(); }
 144 
 145 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 146 
 147 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 148 
 149 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
 150 //-------------------------------------------
 151 
 152 static Register as_reg(LIR_Opr op) {
 153   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 154 }
 155 
 156 static jlong as_long(LIR_Opr data) {
 157   jlong result;
 158   switch (data->type()) {
 159   case T_INT:
 160     result = (data->as_jint());
 161     break;
 162   case T_LONG:
 163     result = (data->as_jlong());
 164     break;
 165   default:
 166     ShouldNotReachHere();
 167     result = 0;  // unreachable
 168   }
 169   return result;
 170 }
 171 
 172 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 173   Register base = addr->base()->as_pointer_register();
 174   LIR_Opr opr = addr->index();
 175   if (opr->is_cpu_register()) {
 176     Register index;
 177     if (opr->is_single_cpu())
 178       index = opr->as_register();
 179     else
 180       index = opr->as_register_lo();
 181     assert(addr->disp() == 0, "must be");
 182     switch(opr->type()) {
 183       case T_INT:
 184         return Address(base, index, Address::sxtw(addr->scale()));
 185       case T_LONG:
 186         return Address(base, index, Address::lsl(addr->scale()));
 187       default:
 188         ShouldNotReachHere();
 189       }
 190   } else {
 191     assert(addr->scale() == 0,
 192            "expected for immediate operand, was: %d", addr->scale());
 193     ptrdiff_t offset = ptrdiff_t(addr->disp());
 194     // NOTE: Does not handle any 16 byte vector access.
 195     const uint type_size = type2aelembytes(addr->type(), true);
 196     return __ legitimize_address(Address(base, offset), type_size, tmp);
 197   }
 198   return Address();
 199 }
 200 
 201 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 202   ShouldNotReachHere();
 203   return Address();
 204 }
 205 
 206 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 207   return as_Address(addr, rscratch1);
 208 }
 209 
 210 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 211   return as_Address(addr, rscratch1);  // Ouch
 212   // FIXME: This needs to be much more clever.  See x86.
 213 }
 214 
 215 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
 216 // not encodable as a base + (immediate) offset, generate an explicit address
 217 // calculation to hold the address in a temporary register.
 218 Address LIR_Assembler::stack_slot_address(int index, uint size, Register tmp, int adjust) {
 219   precond(size == 4 || size == 8);
 220   Address addr = frame_map()->address_for_slot(index, adjust);
 221   precond(addr.getMode() == Address::base_plus_offset);
 222   precond(addr.base() == sp);
 223   precond(addr.offset() > 0);
 224   uint mask = size - 1;
 225   assert((addr.offset() & mask) == 0, "scaled offsets only");
 226   return __ legitimize_address(addr, size, tmp);
 227 }
 228 
 229 void LIR_Assembler::osr_entry() {
 230   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 231   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 232   ValueStack* entry_state = osr_entry->state();
 233   int number_of_locks = entry_state->locks_size();
 234 
 235   // we jump here if osr happens with the interpreter
 236   // state set up to continue at the beginning of the
 237   // loop that triggered osr - in particular, we have
 238   // the following registers setup:
 239   //
 240   // r2: osr buffer
 241   //
 242 
 243   // build frame
 244   ciMethod* m = compilation()->method();
 245   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 246 
 247   // OSR buffer is
 248   //
 249   // locals[nlocals-1..0]
 250   // monitors[0..number_of_locks]
 251   //
 252   // locals is a direct copy of the interpreter frame so in the osr buffer
 253   // so first slot in the local array is the last local from the interpreter
 254   // and last slot is local[0] (receiver) from the interpreter
 255   //
 256   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 257   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 258   // in the interpreter frame (the method lock if a sync method)
 259 
 260   // Initialize monitors in the compiled activation.
 261   //   r2: pointer to osr buffer
 262   //
 263   // All other registers are dead at this point and the locals will be
 264   // copied into place by code emitted in the IR.
 265 
 266   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 267   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 268     int monitor_offset = BytesPerWord * method()->max_locals() +
 269       BytesPerWord * (number_of_locks - 1);
 270     for (int i = 0; i < number_of_locks; i++) {
 271       int slot_offset = monitor_offset - (i * BytesPerWord);
 272 #ifdef ASSERT
 273       // verify the interpreter's monitor has a non-null object
 274       {
 275         Label L;
 276         __ ldr(rscratch1, Address(OSR_buf, slot_offset));
 277         __ cbnz(rscratch1, L);
 278         __ stop("locked object is NULL");
 279         __ bind(L);
 280       }
 281 #endif
 282       __ ldr(r19, Address(OSR_buf, slot_offset));
 283       __ str(r19, frame_map()->address_for_monitor_object(i));
 284     }
 285   }
 286 }
 287 
 288 
 289 // inline cache check; done before the frame is built.
 290 int LIR_Assembler::check_icache() {
 291   Register receiver = FrameMap::receiver_opr->as_register();
 292   Register ic_klass = IC_Klass;
 293   int start_offset = __ offset();
 294   __ inline_cache_check(receiver, ic_klass);
 295 
 296   // if icache check fails, then jump to runtime routine
 297   // Note: RECEIVER must still contain the receiver!
 298   Label dont;
 299   __ br(Assembler::EQ, dont);
 300   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 301 
 302   // We align the verified entry point unless the method body
 303   // (including its inline cache check) will fit in a single 64-byte
 304   // icache line.
 305   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 306     // force alignment after the cache check.
 307     __ align(CodeEntryAlignment);
 308   }
 309 
 310   __ bind(dont);
 311   return start_offset;
 312 }
 313 
 314 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 315   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 316   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 317 
 318   Label L_skip_barrier;
 319 
 320   __ mov_metadata(rscratch2, method->holder()->constant_encoding());
 321   __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier /*L_fast_path*/);
 322   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 323   __ bind(L_skip_barrier);
 324 }
 325 
 326 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 327   if (o == NULL) {
 328     __ mov(reg, zr);
 329   } else {
 330     __ movoop(reg, o);
 331   }
 332 }
 333 
 334 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 335   address target = NULL;
 336   relocInfo::relocType reloc_type = relocInfo::none;
 337 
 338   switch (patching_id(info)) {
 339   case PatchingStub::access_field_id:
 340     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 341     reloc_type = relocInfo::section_word_type;
 342     break;
 343   case PatchingStub::load_klass_id:
 344     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 345     reloc_type = relocInfo::metadata_type;
 346     break;
 347   case PatchingStub::load_mirror_id:
 348     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 349     reloc_type = relocInfo::oop_type;
 350     break;
 351   case PatchingStub::load_appendix_id:
 352     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 353     reloc_type = relocInfo::oop_type;
 354     break;
 355   default: ShouldNotReachHere();
 356   }
 357 
 358   __ far_call(RuntimeAddress(target));
 359   add_call_info_here(info);
 360 }
 361 
 362 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 363   deoptimize_trap(info);
 364 }
 365 
 366 
 367 // This specifies the rsp decrement needed to build the frame
 368 int LIR_Assembler::initial_frame_size_in_bytes() const {
 369   // if rounding, must let FrameMap know!
 370 
 371   return in_bytes(frame_map()->framesize_in_bytes());
 372 }
 373 
 374 
 375 int LIR_Assembler::emit_exception_handler() {
 376   // generate code for exception handler
 377   address handler_base = __ start_a_stub(exception_handler_size());
 378   if (handler_base == NULL) {
 379     // not enough space left for the handler
 380     bailout("exception handler overflow");
 381     return -1;
 382   }
 383 
 384   int offset = code_offset();
 385 
 386   // the exception oop and pc are in r0, and r3
 387   // no other registers need to be preserved, so invalidate them
 388   __ invalidate_registers(false, true, true, false, true, true);
 389 
 390   // check that there is really an exception
 391   __ verify_not_null_oop(r0);
 392 
 393   // search an exception handler (r0: exception oop, r3: throwing pc)
 394   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 395   __ should_not_reach_here();
 396   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 397   __ end_a_stub();
 398 
 399   return offset;
 400 }
 401 
 402 
 403 // Emit the code to remove the frame from the stack in the exception
 404 // unwind path.
 405 int LIR_Assembler::emit_unwind_handler() {
 406 #ifndef PRODUCT
 407   if (CommentedAssembly) {
 408     _masm->block_comment("Unwind handler");
 409   }
 410 #endif
 411 
 412   int offset = code_offset();
 413 
 414   // Fetch the exception from TLS and clear out exception related thread state
 415   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 416   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 417   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 418 
 419   __ bind(_unwind_handler_entry);
 420   __ verify_not_null_oop(r0);
 421   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 422     __ mov(r19, r0);  // Preserve the exception
 423   }
 424 
 425   // Perform needed unlocking
 426   MonitorExitStub* stub = NULL;
 427   if (method()->is_synchronized()) {
 428     monitor_address(0, FrameMap::r0_opr);
 429     __ ldr(r4, Address(r0, BasicObjectLock::obj_offset_in_bytes()));
 430     stub = new MonitorExitStub(FrameMap::r4_opr);
 431     if (UseHeavyMonitors) {
 432       __ b(*stub->entry());
 433     } else {
 434       __ unlock_object(r5, r4, r0, *stub->entry());
 435     }
 436     __ bind(*stub->continuation());
 437   }
 438 
 439   if (compilation()->env()->dtrace_method_probes()) {
 440     __ mov(c_rarg0, rthread);
 441     __ mov_metadata(c_rarg1, method()->constant_encoding());
 442     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 443   }
 444 
 445   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 446     __ mov(r0, r19);  // Restore the exception
 447   }
 448 
 449   // remove the activation and dispatch to the unwind handler
 450   __ block_comment("remove_frame and dispatch to the unwind handler");
 451   __ remove_frame(initial_frame_size_in_bytes());
 452   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 453 
 454   // Emit the slow path assembly
 455   if (stub != NULL) {
 456     stub->emit_code(this);
 457   }
 458 
 459   return offset;
 460 }
 461 
 462 
 463 int LIR_Assembler::emit_deopt_handler() {
 464   // generate code for exception handler
 465   address handler_base = __ start_a_stub(deopt_handler_size());
 466   if (handler_base == NULL) {
 467     // not enough space left for the handler
 468     bailout("deopt handler overflow");
 469     return -1;
 470   }
 471 
 472   int offset = code_offset();
 473 
 474   __ adr(lr, pc());
 475   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 476   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 477   __ end_a_stub();
 478 
 479   return offset;
 480 }
 481 
 482 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 483   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 484   int pc_offset = code_offset();
 485   flush_debug_info(pc_offset);
 486   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 487   if (info->exception_handlers() != NULL) {
 488     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 489   }
 490 }
 491 
 492 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 493   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 494 
 495   // Pop the stack before the safepoint code
 496   __ remove_frame(initial_frame_size_in_bytes());
 497 
 498   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 499     __ reserved_stack_check();
 500   }
 501 
 502   code_stub->set_safepoint_offset(__ offset());
 503   __ relocate(relocInfo::poll_return_type);
 504   __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
 505   __ ret(lr);
 506 }
 507 
 508 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 509   guarantee(info != NULL, "Shouldn't be NULL");
 510   __ get_polling_page(rscratch1, relocInfo::poll_type);
 511   add_debug_info_for_branch(info);  // This isn't just debug info:
 512                                     // it's the oop map
 513   __ read_polling_page(rscratch1, relocInfo::poll_type);
 514   return __ offset();
 515 }
 516 
 517 
 518 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 519   if (from_reg == r31_sp)
 520     from_reg = sp;
 521   if (to_reg == r31_sp)
 522     to_reg = sp;
 523   __ mov(to_reg, from_reg);
 524 }
 525 
 526 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 527 
 528 
 529 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 530   assert(src->is_constant(), "should not call otherwise");
 531   assert(dest->is_register(), "should not call otherwise");
 532   LIR_Const* c = src->as_constant_ptr();
 533 
 534   switch (c->type()) {
 535     case T_INT: {
 536       assert(patch_code == lir_patch_none, "no patching handled here");
 537       __ movw(dest->as_register(), c->as_jint());
 538       break;
 539     }
 540 
 541     case T_ADDRESS: {
 542       assert(patch_code == lir_patch_none, "no patching handled here");
 543       __ mov(dest->as_register(), c->as_jint());
 544       break;
 545     }
 546 
 547     case T_LONG: {
 548       assert(patch_code == lir_patch_none, "no patching handled here");
 549       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
 550       break;
 551     }
 552 
 553     case T_OBJECT: {
 554         if (patch_code == lir_patch_none) {
 555           jobject2reg(c->as_jobject(), dest->as_register());
 556         } else {
 557           jobject2reg_with_patching(dest->as_register(), info);
 558         }
 559       break;
 560     }
 561 
 562     case T_METADATA: {
 563       if (patch_code != lir_patch_none) {
 564         klass2reg_with_patching(dest->as_register(), info);
 565       } else {
 566         __ mov_metadata(dest->as_register(), c->as_metadata());
 567       }
 568       break;
 569     }
 570 
 571     case T_FLOAT: {
 572       if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 573         __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
 574       } else {
 575         __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 576         __ ldrs(dest->as_float_reg(), Address(rscratch1));
 577       }
 578       break;
 579     }
 580 
 581     case T_DOUBLE: {
 582       if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
 583         __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
 584       } else {
 585         __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 586         __ ldrd(dest->as_double_reg(), Address(rscratch1));
 587       }
 588       break;
 589     }
 590 
 591     default:
 592       ShouldNotReachHere();
 593   }
 594 }
 595 
 596 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 597   LIR_Const* c = src->as_constant_ptr();
 598   switch (c->type()) {
 599   case T_OBJECT:
 600     {
 601       if (! c->as_jobject())
 602         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 603       else {
 604         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 605         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 606       }
 607     }
 608     break;
 609   case T_ADDRESS:
 610     {
 611       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 612       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 613     }
 614   case T_INT:
 615   case T_FLOAT:
 616     {
 617       Register reg = zr;
 618       if (c->as_jint_bits() == 0)
 619         __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 620       else {
 621         __ movw(rscratch1, c->as_jint_bits());
 622         __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 623       }
 624     }
 625     break;
 626   case T_LONG:
 627   case T_DOUBLE:
 628     {
 629       Register reg = zr;
 630       if (c->as_jlong_bits() == 0)
 631         __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 632                                                  lo_word_offset_in_bytes));
 633       else {
 634         __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
 635         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 636                                                         lo_word_offset_in_bytes));
 637       }
 638     }
 639     break;
 640   default:
 641     ShouldNotReachHere();
 642   }
 643 }
 644 
 645 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 646   assert(src->is_constant(), "should not call otherwise");
 647   LIR_Const* c = src->as_constant_ptr();
 648   LIR_Address* to_addr = dest->as_address_ptr();
 649 
 650   void (Assembler::* insn)(Register Rt, const Address &adr);
 651 
 652   switch (type) {
 653   case T_ADDRESS:
 654     assert(c->as_jint() == 0, "should be");
 655     insn = &Assembler::str;
 656     break;
 657   case T_LONG:
 658     assert(c->as_jlong() == 0, "should be");
 659     insn = &Assembler::str;
 660     break;
 661   case T_INT:
 662     assert(c->as_jint() == 0, "should be");
 663     insn = &Assembler::strw;
 664     break;
 665   case T_OBJECT:
 666   case T_ARRAY:
 667     assert(c->as_jobject() == 0, "should be");
 668     if (UseCompressedOops && !wide) {
 669       insn = &Assembler::strw;
 670     } else {
 671       insn = &Assembler::str;
 672     }
 673     break;
 674   case T_CHAR:
 675   case T_SHORT:
 676     assert(c->as_jint() == 0, "should be");
 677     insn = &Assembler::strh;
 678     break;
 679   case T_BOOLEAN:
 680   case T_BYTE:
 681     assert(c->as_jint() == 0, "should be");
 682     insn = &Assembler::strb;
 683     break;
 684   default:
 685     ShouldNotReachHere();
 686     insn = &Assembler::str;  // unreachable
 687   }
 688 
 689   if (info) add_debug_info_for_null_check_here(info);
 690   (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
 691 }
 692 
 693 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 694   assert(src->is_register(), "should not call otherwise");
 695   assert(dest->is_register(), "should not call otherwise");
 696 
 697   // move between cpu-registers
 698   if (dest->is_single_cpu()) {
 699     if (src->type() == T_LONG) {
 700       // Can do LONG -> OBJECT
 701       move_regs(src->as_register_lo(), dest->as_register());
 702       return;
 703     }
 704     assert(src->is_single_cpu(), "must match");
 705     if (src->type() == T_OBJECT) {
 706       __ verify_oop(src->as_register());
 707     }
 708     move_regs(src->as_register(), dest->as_register());
 709 
 710   } else if (dest->is_double_cpu()) {
 711     if (is_reference_type(src->type())) {
 712       // Surprising to me but we can see move of a long to t_object
 713       __ verify_oop(src->as_register());
 714       move_regs(src->as_register(), dest->as_register_lo());
 715       return;
 716     }
 717     assert(src->is_double_cpu(), "must match");
 718     Register f_lo = src->as_register_lo();
 719     Register f_hi = src->as_register_hi();
 720     Register t_lo = dest->as_register_lo();
 721     Register t_hi = dest->as_register_hi();
 722     assert(f_hi == f_lo, "must be same");
 723     assert(t_hi == t_lo, "must be same");
 724     move_regs(f_lo, t_lo);
 725 
 726   } else if (dest->is_single_fpu()) {
 727     __ fmovs(dest->as_float_reg(), src->as_float_reg());
 728 
 729   } else if (dest->is_double_fpu()) {
 730     __ fmovd(dest->as_double_reg(), src->as_double_reg());
 731 
 732   } else {
 733     ShouldNotReachHere();
 734   }
 735 }
 736 
 737 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 738   precond(src->is_register() && dest->is_stack());
 739 
 740   uint const c_sz32 = sizeof(uint32_t);
 741   uint const c_sz64 = sizeof(uint64_t);
 742 
 743   if (src->is_single_cpu()) {
 744     int index = dest->single_stack_ix();
 745     if (is_reference_type(type)) {
 746       __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 747       __ verify_oop(src->as_register());
 748     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 749       __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 750     } else {
 751       __ strw(src->as_register(), stack_slot_address(index, c_sz32, rscratch1));
 752     }
 753 
 754   } else if (src->is_double_cpu()) {
 755     int index = dest->double_stack_ix();
 756     Address dest_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
 757     __ str(src->as_register_lo(), dest_addr_LO);
 758 
 759   } else if (src->is_single_fpu()) {
 760     int index = dest->single_stack_ix();
 761     __ strs(src->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
 762 
 763   } else if (src->is_double_fpu()) {
 764     int index = dest->double_stack_ix();
 765     __ strd(src->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
 766 
 767   } else {
 768     ShouldNotReachHere();
 769   }
 770 }
 771 
 772 
 773 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
 774   LIR_Address* to_addr = dest->as_address_ptr();
 775   PatchingStub* patch = NULL;
 776   Register compressed_src = rscratch1;
 777 
 778   if (patch_code != lir_patch_none) {
 779     deoptimize_trap(info);
 780     return;
 781   }
 782 
 783   if (is_reference_type(type)) {
 784     __ verify_oop(src->as_register());
 785 
 786     if (UseCompressedOops && !wide) {
 787       __ encode_heap_oop(compressed_src, src->as_register());
 788     } else {
 789       compressed_src = src->as_register();
 790     }
 791   }
 792 
 793   int null_check_here = code_offset();
 794   switch (type) {
 795     case T_FLOAT: {
 796       __ strs(src->as_float_reg(), as_Address(to_addr));
 797       break;
 798     }
 799 
 800     case T_DOUBLE: {
 801       __ strd(src->as_double_reg(), as_Address(to_addr));
 802       break;
 803     }
 804 
 805     case T_ARRAY:   // fall through
 806     case T_OBJECT:  // fall through
 807       if (UseCompressedOops && !wide) {
 808         __ strw(compressed_src, as_Address(to_addr, rscratch2));
 809       } else {
 810          __ str(compressed_src, as_Address(to_addr));
 811       }
 812       break;
 813     case T_METADATA:
 814       // We get here to store a method pointer to the stack to pass to
 815       // a dtrace runtime call. This can't work on 64 bit with
 816       // compressed klass ptrs: T_METADATA can be a compressed klass
 817       // ptr or a 64 bit method pointer.
 818       ShouldNotReachHere();
 819       __ str(src->as_register(), as_Address(to_addr));
 820       break;
 821     case T_ADDRESS:
 822       __ str(src->as_register(), as_Address(to_addr));
 823       break;
 824     case T_INT:
 825       __ strw(src->as_register(), as_Address(to_addr));
 826       break;
 827 
 828     case T_LONG: {
 829       __ str(src->as_register_lo(), as_Address_lo(to_addr));
 830       break;
 831     }
 832 
 833     case T_BYTE:    // fall through
 834     case T_BOOLEAN: {
 835       __ strb(src->as_register(), as_Address(to_addr));
 836       break;
 837     }
 838 
 839     case T_CHAR:    // fall through
 840     case T_SHORT:
 841       __ strh(src->as_register(), as_Address(to_addr));
 842       break;
 843 
 844     default:
 845       ShouldNotReachHere();
 846   }
 847   if (info != NULL) {
 848     add_debug_info_for_null_check(null_check_here, info);
 849   }
 850 }
 851 
 852 
 853 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 854   precond(src->is_stack() && dest->is_register());
 855 
 856   uint const c_sz32 = sizeof(uint32_t);
 857   uint const c_sz64 = sizeof(uint64_t);
 858 
 859   if (dest->is_single_cpu()) {
 860     int index = src->single_stack_ix();
 861     if (is_reference_type(type)) {
 862       __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 863       __ verify_oop(dest->as_register());
 864     } else if (type == T_METADATA || type == T_ADDRESS) {
 865       __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 866     } else {
 867       __ ldrw(dest->as_register(), stack_slot_address(index, c_sz32, rscratch1));
 868     }
 869 
 870   } else if (dest->is_double_cpu()) {
 871     int index = src->double_stack_ix();
 872     Address src_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
 873     __ ldr(dest->as_register_lo(), src_addr_LO);
 874 
 875   } else if (dest->is_single_fpu()) {
 876     int index = src->single_stack_ix();
 877     __ ldrs(dest->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
 878 
 879   } else if (dest->is_double_fpu()) {
 880     int index = src->double_stack_ix();
 881     __ ldrd(dest->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
 882 
 883   } else {
 884     ShouldNotReachHere();
 885   }
 886 }
 887 
 888 
 889 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 890   address target = NULL;
 891   relocInfo::relocType reloc_type = relocInfo::none;
 892 
 893   switch (patching_id(info)) {
 894   case PatchingStub::access_field_id:
 895     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 896     reloc_type = relocInfo::section_word_type;
 897     break;
 898   case PatchingStub::load_klass_id:
 899     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 900     reloc_type = relocInfo::metadata_type;
 901     break;
 902   case PatchingStub::load_mirror_id:
 903     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 904     reloc_type = relocInfo::oop_type;
 905     break;
 906   case PatchingStub::load_appendix_id:
 907     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 908     reloc_type = relocInfo::oop_type;
 909     break;
 910   default: ShouldNotReachHere();
 911   }
 912 
 913   __ far_call(RuntimeAddress(target));
 914   add_call_info_here(info);
 915 }
 916 
 917 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 918 
 919   LIR_Opr temp;
 920   if (type == T_LONG || type == T_DOUBLE)
 921     temp = FrameMap::rscratch1_long_opr;
 922   else
 923     temp = FrameMap::rscratch1_opr;
 924 
 925   stack2reg(src, temp, src->type());
 926   reg2stack(temp, dest, dest->type(), false);
 927 }
 928 
 929 
 930 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 931   LIR_Address* addr = src->as_address_ptr();
 932   LIR_Address* from_addr = src->as_address_ptr();
 933 
 934   if (addr->base()->type() == T_OBJECT) {
 935     __ verify_oop(addr->base()->as_pointer_register());
 936   }
 937 
 938   if (patch_code != lir_patch_none) {
 939     deoptimize_trap(info);
 940     return;
 941   }
 942 
 943   if (info != NULL) {
 944     add_debug_info_for_null_check_here(info);
 945   }
 946   int null_check_here = code_offset();
 947   switch (type) {
 948     case T_FLOAT: {
 949       __ ldrs(dest->as_float_reg(), as_Address(from_addr));
 950       break;
 951     }
 952 
 953     case T_DOUBLE: {
 954       __ ldrd(dest->as_double_reg(), as_Address(from_addr));
 955       break;
 956     }
 957 
 958     case T_ARRAY:   // fall through
 959     case T_OBJECT:  // fall through
 960       if (UseCompressedOops && !wide) {
 961         __ ldrw(dest->as_register(), as_Address(from_addr));
 962       } else {
 963         __ ldr(dest->as_register(), as_Address(from_addr));
 964       }
 965       break;
 966     case T_METADATA:
 967       // We get here to store a method pointer to the stack to pass to
 968       // a dtrace runtime call. This can't work on 64 bit with
 969       // compressed klass ptrs: T_METADATA can be a compressed klass
 970       // ptr or a 64 bit method pointer.
 971       ShouldNotReachHere();
 972       __ ldr(dest->as_register(), as_Address(from_addr));
 973       break;
 974     case T_ADDRESS:
 975       __ ldr(dest->as_register(), as_Address(from_addr));
 976       break;
 977     case T_INT:
 978       __ ldrw(dest->as_register(), as_Address(from_addr));
 979       break;
 980 
 981     case T_LONG: {
 982       __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
 983       break;
 984     }
 985 
 986     case T_BYTE:
 987       __ ldrsb(dest->as_register(), as_Address(from_addr));
 988       break;
 989     case T_BOOLEAN: {
 990       __ ldrb(dest->as_register(), as_Address(from_addr));
 991       break;
 992     }
 993 
 994     case T_CHAR:
 995       __ ldrh(dest->as_register(), as_Address(from_addr));
 996       break;
 997     case T_SHORT:
 998       __ ldrsh(dest->as_register(), as_Address(from_addr));
 999       break;
1000 
1001     default:
1002       ShouldNotReachHere();
1003   }
1004 
1005   if (is_reference_type(type)) {
1006     if (UseCompressedOops && !wide) {
1007       __ decode_heap_oop(dest->as_register());
1008     }
1009 
1010     if (!UseZGC) {
1011       // Load barrier has not yet been applied, so ZGC can't verify the oop here
1012       __ verify_oop(dest->as_register());
1013     }
1014   }
1015 }
1016 
1017 
1018 int LIR_Assembler::array_element_size(BasicType type) const {
1019   int elem_size = type2aelembytes(type);
1020   return exact_log2(elem_size);
1021 }
1022 
1023 
1024 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1025   switch (op->code()) {
1026   case lir_idiv:
1027   case lir_irem:
1028     arithmetic_idiv(op->code(),
1029                     op->in_opr1(),
1030                     op->in_opr2(),
1031                     op->in_opr3(),
1032                     op->result_opr(),
1033                     op->info());
1034     break;
1035   case lir_fmad:
1036     __ fmaddd(op->result_opr()->as_double_reg(),
1037               op->in_opr1()->as_double_reg(),
1038               op->in_opr2()->as_double_reg(),
1039               op->in_opr3()->as_double_reg());
1040     break;
1041   case lir_fmaf:
1042     __ fmadds(op->result_opr()->as_float_reg(),
1043               op->in_opr1()->as_float_reg(),
1044               op->in_opr2()->as_float_reg(),
1045               op->in_opr3()->as_float_reg());
1046     break;
1047   default:      ShouldNotReachHere(); break;
1048   }
1049 }
1050 
1051 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1052 #ifdef ASSERT
1053   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1054   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1055   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1056 #endif
1057 
1058   if (op->cond() == lir_cond_always) {
1059     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1060     __ b(*(op->label()));
1061   } else {
1062     Assembler::Condition acond;
1063     if (op->code() == lir_cond_float_branch) {
1064       bool is_unordered = (op->ublock() == op->block());
1065       // Assembler::EQ does not permit unordered branches, so we add
1066       // another branch here.  Likewise, Assembler::NE does not permit
1067       // ordered branches.
1068       if ((is_unordered && op->cond() == lir_cond_equal)
1069           || (!is_unordered && op->cond() == lir_cond_notEqual))
1070         __ br(Assembler::VS, *(op->ublock()->label()));
1071       switch(op->cond()) {
1072       case lir_cond_equal:        acond = Assembler::EQ; break;
1073       case lir_cond_notEqual:     acond = Assembler::NE; break;
1074       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1075       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1076       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1077       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1078       default:                    ShouldNotReachHere();
1079         acond = Assembler::EQ;  // unreachable
1080       }
1081     } else {
1082       switch (op->cond()) {
1083         case lir_cond_equal:        acond = Assembler::EQ; break;
1084         case lir_cond_notEqual:     acond = Assembler::NE; break;
1085         case lir_cond_less:         acond = Assembler::LT; break;
1086         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1087         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1088         case lir_cond_greater:      acond = Assembler::GT; break;
1089         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1090         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1091         default:                    ShouldNotReachHere();
1092           acond = Assembler::EQ;  // unreachable
1093       }
1094     }
1095     __ br(acond,*(op->label()));
1096   }
1097 }
1098 
1099 
1100 
1101 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1102   LIR_Opr src  = op->in_opr();
1103   LIR_Opr dest = op->result_opr();
1104 
1105   switch (op->bytecode()) {
1106     case Bytecodes::_i2f:
1107       {
1108         __ scvtfws(dest->as_float_reg(), src->as_register());
1109         break;
1110       }
1111     case Bytecodes::_i2d:
1112       {
1113         __ scvtfwd(dest->as_double_reg(), src->as_register());
1114         break;
1115       }
1116     case Bytecodes::_l2d:
1117       {
1118         __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1119         break;
1120       }
1121     case Bytecodes::_l2f:
1122       {
1123         __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1124         break;
1125       }
1126     case Bytecodes::_f2d:
1127       {
1128         __ fcvts(dest->as_double_reg(), src->as_float_reg());
1129         break;
1130       }
1131     case Bytecodes::_d2f:
1132       {
1133         __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1134         break;
1135       }
1136     case Bytecodes::_i2c:
1137       {
1138         __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1139         break;
1140       }
1141     case Bytecodes::_i2l:
1142       {
1143         __ sxtw(dest->as_register_lo(), src->as_register());
1144         break;
1145       }
1146     case Bytecodes::_i2s:
1147       {
1148         __ sxth(dest->as_register(), src->as_register());
1149         break;
1150       }
1151     case Bytecodes::_i2b:
1152       {
1153         __ sxtb(dest->as_register(), src->as_register());
1154         break;
1155       }
1156     case Bytecodes::_l2i:
1157       {
1158         _masm->block_comment("FIXME: This could be a no-op");
1159         __ uxtw(dest->as_register(), src->as_register_lo());
1160         break;
1161       }
1162     case Bytecodes::_d2l:
1163       {
1164         __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1165         break;
1166       }
1167     case Bytecodes::_f2i:
1168       {
1169         __ fcvtzsw(dest->as_register(), src->as_float_reg());
1170         break;
1171       }
1172     case Bytecodes::_f2l:
1173       {
1174         __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1175         break;
1176       }
1177     case Bytecodes::_d2i:
1178       {
1179         __ fcvtzdw(dest->as_register(), src->as_double_reg());
1180         break;
1181       }
1182     default: ShouldNotReachHere();
1183   }
1184 }
1185 
1186 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1187   if (op->init_check()) {
1188     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1189                                InstanceKlass::init_state_offset()));
1190     __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1191     add_debug_info_for_null_check_here(op->stub()->info());
1192     __ br(Assembler::NE, *op->stub()->entry());
1193   }
1194   __ allocate_object(op->obj()->as_register(),
1195                      op->tmp1()->as_register(),
1196                      op->tmp2()->as_register(),
1197                      op->header_size(),
1198                      op->object_size(),
1199                      op->klass()->as_register(),
1200                      *op->stub()->entry());
1201   __ bind(*op->stub()->continuation());
1202 }
1203 
1204 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1205   Register len =  op->len()->as_register();
1206   __ uxtw(len, len);
1207 
1208   if (UseSlowPath ||
1209       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1210       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1211     __ b(*op->stub()->entry());
1212   } else {
1213     Register tmp1 = op->tmp1()->as_register();
1214     Register tmp2 = op->tmp2()->as_register();
1215     Register tmp3 = op->tmp3()->as_register();
1216     if (len == tmp1) {
1217       tmp1 = tmp3;
1218     } else if (len == tmp2) {
1219       tmp2 = tmp3;
1220     } else if (len == tmp3) {
1221       // everything is ok
1222     } else {
1223       __ mov(tmp3, len);
1224     }
1225     __ allocate_array(op->obj()->as_register(),
1226                       len,
1227                       tmp1,
1228                       tmp2,
1229                       arrayOopDesc::base_offset_in_bytes(op->type()),
1230                       array_element_size(op->type()),
1231                       op->klass()->as_register(),
1232                       *op->stub()->entry());
1233   }
1234   __ bind(*op->stub()->continuation());
1235 }
1236 
1237 void LIR_Assembler::type_profile_helper(Register mdo,
1238                                         ciMethodData *md, ciProfileData *data,
1239                                         Register recv, Label* update_done) {
1240   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1241     Label next_test;
1242     // See if the receiver is receiver[n].
1243     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1244     __ ldr(rscratch1, Address(rscratch2));
1245     __ cmp(recv, rscratch1);
1246     __ br(Assembler::NE, next_test);
1247     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1248     __ addptr(data_addr, DataLayout::counter_increment);
1249     __ b(*update_done);
1250     __ bind(next_test);
1251   }
1252 
1253   // Didn't find receiver; find next empty slot and fill it in
1254   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1255     Label next_test;
1256     __ lea(rscratch2,
1257            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1258     Address recv_addr(rscratch2);
1259     __ ldr(rscratch1, recv_addr);
1260     __ cbnz(rscratch1, next_test);
1261     __ str(recv, recv_addr);
1262     __ mov(rscratch1, DataLayout::counter_increment);
1263     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1264     __ str(rscratch1, Address(rscratch2));
1265     __ b(*update_done);
1266     __ bind(next_test);
1267   }
1268 }
1269 
1270 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1271   // we always need a stub for the failure case.
1272   CodeStub* stub = op->stub();
1273   Register obj = op->object()->as_register();
1274   Register k_RInfo = op->tmp1()->as_register();
1275   Register klass_RInfo = op->tmp2()->as_register();
1276   Register dst = op->result_opr()->as_register();
1277   ciKlass* k = op->klass();
1278   Register Rtmp1 = noreg;
1279 
1280   // check if it needs to be profiled
1281   ciMethodData* md;
1282   ciProfileData* data;
1283 
1284   const bool should_profile = op->should_profile();
1285 
1286   if (should_profile) {
1287     ciMethod* method = op->profiled_method();
1288     assert(method != NULL, "Should have method");
1289     int bci = op->profiled_bci();
1290     md = method->method_data_or_null();
1291     assert(md != NULL, "Sanity");
1292     data = md->bci_to_data(bci);
1293     assert(data != NULL,                "need data for type check");
1294     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1295   }
1296   Label profile_cast_success, profile_cast_failure;
1297   Label *success_target = should_profile ? &profile_cast_success : success;
1298   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1299 
1300   if (obj == k_RInfo) {
1301     k_RInfo = dst;
1302   } else if (obj == klass_RInfo) {
1303     klass_RInfo = dst;
1304   }
1305   if (k->is_loaded() && !UseCompressedClassPointers) {
1306     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1307   } else {
1308     Rtmp1 = op->tmp3()->as_register();
1309     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1310   }
1311 
1312   assert_different_registers(obj, k_RInfo, klass_RInfo);
1313 
1314     if (should_profile) {
1315       Label not_null;
1316       __ cbnz(obj, not_null);
1317       // Object is null; update MDO and exit
1318       Register mdo  = klass_RInfo;
1319       __ mov_metadata(mdo, md->constant_encoding());
1320       Address data_addr
1321         = __ form_address(rscratch2, mdo,
1322                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1323                           0);
1324       __ ldrb(rscratch1, data_addr);
1325       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1326       __ strb(rscratch1, data_addr);
1327       __ b(*obj_is_null);
1328       __ bind(not_null);
1329     } else {
1330       __ cbz(obj, *obj_is_null);
1331     }
1332 
1333   if (!k->is_loaded()) {
1334     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1335   } else {
1336     __ mov_metadata(k_RInfo, k->constant_encoding());
1337   }
1338   __ verify_oop(obj);
1339 
1340   if (op->fast_check()) {
1341     // get object class
1342     // not a safepoint as obj null check happens earlier
1343     __ load_klass(rscratch1, obj);
1344     __ cmp( rscratch1, k_RInfo);
1345 
1346     __ br(Assembler::NE, *failure_target);
1347     // successful cast, fall through to profile or jump
1348   } else {
1349     // get object class
1350     // not a safepoint as obj null check happens earlier
1351     __ load_klass(klass_RInfo, obj);
1352     if (k->is_loaded()) {
1353       // See if we get an immediate positive hit
1354       __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1355       __ cmp(k_RInfo, rscratch1);
1356       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1357         __ br(Assembler::NE, *failure_target);
1358         // successful cast, fall through to profile or jump
1359       } else {
1360         // See if we get an immediate positive hit
1361         __ br(Assembler::EQ, *success_target);
1362         // check for self
1363         __ cmp(klass_RInfo, k_RInfo);
1364         __ br(Assembler::EQ, *success_target);
1365 
1366         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1367         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1368         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1369         // result is a boolean
1370         __ cbzw(klass_RInfo, *failure_target);
1371         // successful cast, fall through to profile or jump
1372       }
1373     } else {
1374       // perform the fast part of the checking logic
1375       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1376       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1377       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1378       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1379       __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1380       // result is a boolean
1381       __ cbz(k_RInfo, *failure_target);
1382       // successful cast, fall through to profile or jump
1383     }
1384   }
1385   if (should_profile) {
1386     Register mdo  = klass_RInfo, recv = k_RInfo;
1387     __ bind(profile_cast_success);
1388     __ mov_metadata(mdo, md->constant_encoding());
1389     __ load_klass(recv, obj);
1390     Label update_done;
1391     type_profile_helper(mdo, md, data, recv, success);
1392     __ b(*success);
1393 
1394     __ bind(profile_cast_failure);
1395     __ mov_metadata(mdo, md->constant_encoding());
1396     Address counter_addr
1397       = __ form_address(rscratch2, mdo,
1398                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1399                         0);
1400     __ ldr(rscratch1, counter_addr);
1401     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1402     __ str(rscratch1, counter_addr);
1403     __ b(*failure);
1404   }
1405   __ b(*success);
1406 }
1407 
1408 
1409 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1410   const bool should_profile = op->should_profile();
1411 
1412   LIR_Code code = op->code();
1413   if (code == lir_store_check) {
1414     Register value = op->object()->as_register();
1415     Register array = op->array()->as_register();
1416     Register k_RInfo = op->tmp1()->as_register();
1417     Register klass_RInfo = op->tmp2()->as_register();
1418     Register Rtmp1 = op->tmp3()->as_register();
1419 
1420     CodeStub* stub = op->stub();
1421 
1422     // check if it needs to be profiled
1423     ciMethodData* md;
1424     ciProfileData* data;
1425 
1426     if (should_profile) {
1427       ciMethod* method = op->profiled_method();
1428       assert(method != NULL, "Should have method");
1429       int bci = op->profiled_bci();
1430       md = method->method_data_or_null();
1431       assert(md != NULL, "Sanity");
1432       data = md->bci_to_data(bci);
1433       assert(data != NULL,                "need data for type check");
1434       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1435     }
1436     Label profile_cast_success, profile_cast_failure, done;
1437     Label *success_target = should_profile ? &profile_cast_success : &done;
1438     Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1439 
1440     if (should_profile) {
1441       Label not_null;
1442       __ cbnz(value, not_null);
1443       // Object is null; update MDO and exit
1444       Register mdo  = klass_RInfo;
1445       __ mov_metadata(mdo, md->constant_encoding());
1446       Address data_addr
1447         = __ form_address(rscratch2, mdo,
1448                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1449                           0);
1450       __ ldrb(rscratch1, data_addr);
1451       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1452       __ strb(rscratch1, data_addr);
1453       __ b(done);
1454       __ bind(not_null);
1455     } else {
1456       __ cbz(value, done);
1457     }
1458 
1459     add_debug_info_for_null_check_here(op->info_for_exception());
1460     __ load_klass(k_RInfo, array);
1461     __ load_klass(klass_RInfo, value);
1462 
1463     // get instance klass (it's already uncompressed)
1464     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1465     // perform the fast part of the checking logic
1466     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1467     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1468     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1469     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1470     __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1471     // result is a boolean
1472     __ cbzw(k_RInfo, *failure_target);
1473     // fall through to the success case
1474 
1475     if (should_profile) {
1476       Register mdo  = klass_RInfo, recv = k_RInfo;
1477       __ bind(profile_cast_success);
1478       __ mov_metadata(mdo, md->constant_encoding());
1479       __ load_klass(recv, value);
1480       Label update_done;
1481       type_profile_helper(mdo, md, data, recv, &done);
1482       __ b(done);
1483 
1484       __ bind(profile_cast_failure);
1485       __ mov_metadata(mdo, md->constant_encoding());
1486       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1487       __ lea(rscratch2, counter_addr);
1488       __ ldr(rscratch1, Address(rscratch2));
1489       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1490       __ str(rscratch1, Address(rscratch2));
1491       __ b(*stub->entry());
1492     }
1493 
1494     __ bind(done);
1495   } else if (code == lir_checkcast) {
1496     Register obj = op->object()->as_register();
1497     Register dst = op->result_opr()->as_register();
1498     Label success;
1499     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1500     __ bind(success);
1501     if (dst != obj) {
1502       __ mov(dst, obj);
1503     }
1504   } else if (code == lir_instanceof) {
1505     Register obj = op->object()->as_register();
1506     Register dst = op->result_opr()->as_register();
1507     Label success, failure, done;
1508     emit_typecheck_helper(op, &success, &failure, &failure);
1509     __ bind(failure);
1510     __ mov(dst, zr);
1511     __ b(done);
1512     __ bind(success);
1513     __ mov(dst, 1);
1514     __ bind(done);
1515   } else {
1516     ShouldNotReachHere();
1517   }
1518 }
1519 
1520 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1521   __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1522   __ cset(rscratch1, Assembler::NE);
1523   __ membar(__ AnyAny);
1524 }
1525 
1526 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1527   __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1528   __ cset(rscratch1, Assembler::NE);
1529   __ membar(__ AnyAny);
1530 }
1531 
1532 
1533 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1534   assert(VM_Version::supports_cx8(), "wrong machine");
1535   Register addr;
1536   if (op->addr()->is_register()) {
1537     addr = as_reg(op->addr());
1538   } else {
1539     assert(op->addr()->is_address(), "what else?");
1540     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1541     assert(addr_ptr->disp() == 0, "need 0 disp");
1542     assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1543     addr = as_reg(addr_ptr->base());
1544   }
1545   Register newval = as_reg(op->new_value());
1546   Register cmpval = as_reg(op->cmp_value());
1547 
1548   if (op->code() == lir_cas_obj) {
1549     if (UseCompressedOops) {
1550       Register t1 = op->tmp1()->as_register();
1551       assert(op->tmp1()->is_valid(), "must be");
1552       __ encode_heap_oop(t1, cmpval);
1553       cmpval = t1;
1554       __ encode_heap_oop(rscratch2, newval);
1555       newval = rscratch2;
1556       casw(addr, newval, cmpval);
1557     } else {
1558       casl(addr, newval, cmpval);
1559     }
1560   } else if (op->code() == lir_cas_int) {
1561     casw(addr, newval, cmpval);
1562   } else {
1563     casl(addr, newval, cmpval);
1564   }
1565 }
1566 
1567 
1568 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1569                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1570   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on aarch64");
1571 
1572   Assembler::Condition acond, ncond;
1573   switch (condition) {
1574   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1575   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1576   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1577   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1578   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1579   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1580   case lir_cond_belowEqual:
1581   case lir_cond_aboveEqual:
1582   default:                    ShouldNotReachHere();
1583     acond = Assembler::EQ; ncond = Assembler::NE;  // unreachable
1584   }
1585 
1586   assert(result->is_single_cpu() || result->is_double_cpu(),
1587          "expect single register for result");
1588   if (opr1->is_constant() && opr2->is_constant()
1589       && opr1->type() == T_INT && opr2->type() == T_INT) {
1590     jint val1 = opr1->as_jint();
1591     jint val2 = opr2->as_jint();
1592     if (val1 == 0 && val2 == 1) {
1593       __ cset(result->as_register(), ncond);
1594       return;
1595     } else if (val1 == 1 && val2 == 0) {
1596       __ cset(result->as_register(), acond);
1597       return;
1598     }
1599   }
1600 
1601   if (opr1->is_constant() && opr2->is_constant()
1602       && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1603     jlong val1 = opr1->as_jlong();
1604     jlong val2 = opr2->as_jlong();
1605     if (val1 == 0 && val2 == 1) {
1606       __ cset(result->as_register_lo(), ncond);
1607       return;
1608     } else if (val1 == 1 && val2 == 0) {
1609       __ cset(result->as_register_lo(), acond);
1610       return;
1611     }
1612   }
1613 
1614   if (opr1->is_stack()) {
1615     stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1616     opr1 = FrameMap::rscratch1_opr;
1617   } else if (opr1->is_constant()) {
1618     LIR_Opr tmp
1619       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1620     const2reg(opr1, tmp, lir_patch_none, NULL);
1621     opr1 = tmp;
1622   }
1623 
1624   if (opr2->is_stack()) {
1625     stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1626     opr2 = FrameMap::rscratch2_opr;
1627   } else if (opr2->is_constant()) {
1628     LIR_Opr tmp
1629       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1630     const2reg(opr2, tmp, lir_patch_none, NULL);
1631     opr2 = tmp;
1632   }
1633 
1634   if (result->type() == T_LONG)
1635     __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1636   else
1637     __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1638 }
1639 
1640 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1641   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1642 
1643   if (left->is_single_cpu()) {
1644     Register lreg = left->as_register();
1645     Register dreg = as_reg(dest);
1646 
1647     if (right->is_single_cpu()) {
1648       // cpu register - cpu register
1649 
1650       assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1651              "should be");
1652       Register rreg = right->as_register();
1653       switch (code) {
1654       case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1655       case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1656       case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1657       default:      ShouldNotReachHere();
1658       }
1659 
1660     } else if (right->is_double_cpu()) {
1661       Register rreg = right->as_register_lo();
1662       // single_cpu + double_cpu: can happen with obj+long
1663       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1664       switch (code) {
1665       case lir_add: __ add(dreg, lreg, rreg); break;
1666       case lir_sub: __ sub(dreg, lreg, rreg); break;
1667       default: ShouldNotReachHere();
1668       }
1669     } else if (right->is_constant()) {
1670       // cpu register - constant
1671       jlong c;
1672 
1673       // FIXME.  This is fugly: we really need to factor all this logic.
1674       switch(right->type()) {
1675       case T_LONG:
1676         c = right->as_constant_ptr()->as_jlong();
1677         break;
1678       case T_INT:
1679       case T_ADDRESS:
1680         c = right->as_constant_ptr()->as_jint();
1681         break;
1682       default:
1683         ShouldNotReachHere();
1684         c = 0;  // unreachable
1685         break;
1686       }
1687 
1688       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1689       if (c == 0 && dreg == lreg) {
1690         COMMENT("effective nop elided");
1691         return;
1692       }
1693       switch(left->type()) {
1694       case T_INT:
1695         switch (code) {
1696         case lir_add: __ addw(dreg, lreg, c); break;
1697         case lir_sub: __ subw(dreg, lreg, c); break;
1698         default: ShouldNotReachHere();
1699         }
1700         break;
1701       case T_OBJECT:
1702       case T_ADDRESS:
1703         switch (code) {
1704         case lir_add: __ add(dreg, lreg, c); break;
1705         case lir_sub: __ sub(dreg, lreg, c); break;
1706         default: ShouldNotReachHere();
1707         }
1708         break;
1709       default:
1710         ShouldNotReachHere();
1711       }
1712     } else {
1713       ShouldNotReachHere();
1714     }
1715 
1716   } else if (left->is_double_cpu()) {
1717     Register lreg_lo = left->as_register_lo();
1718 
1719     if (right->is_double_cpu()) {
1720       // cpu register - cpu register
1721       Register rreg_lo = right->as_register_lo();
1722       switch (code) {
1723       case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1724       case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1725       case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1726       case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1727       case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1728       default:
1729         ShouldNotReachHere();
1730       }
1731 
1732     } else if (right->is_constant()) {
1733       jlong c = right->as_constant_ptr()->as_jlong();
1734       Register dreg = as_reg(dest);
1735       switch (code) {
1736         case lir_add:
1737         case lir_sub:
1738           if (c == 0 && dreg == lreg_lo) {
1739             COMMENT("effective nop elided");
1740             return;
1741           }
1742           code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c);
1743           break;
1744         case lir_div:
1745           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1746           if (c == 1) {
1747             // move lreg_lo to dreg if divisor is 1
1748             __ mov(dreg, lreg_lo);
1749           } else {
1750             unsigned int shift = log2i_exact(c);
1751             // use rscratch1 as intermediate result register
1752             __ asr(rscratch1, lreg_lo, 63);
1753             __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift);
1754             __ asr(dreg, rscratch1, shift);
1755           }
1756           break;
1757         case lir_rem:
1758           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1759           if (c == 1) {
1760             // move 0 to dreg if divisor is 1
1761             __ mov(dreg, zr);
1762           } else {
1763             // use rscratch1 as intermediate result register
1764             __ negs(rscratch1, lreg_lo);
1765             __ andr(dreg, lreg_lo, c - 1);
1766             __ andr(rscratch1, rscratch1, c - 1);
1767             __ csneg(dreg, dreg, rscratch1, Assembler::MI);
1768           }
1769           break;
1770         default:
1771           ShouldNotReachHere();
1772       }
1773     } else {
1774       ShouldNotReachHere();
1775     }
1776   } else if (left->is_single_fpu()) {
1777     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1778     switch (code) {
1779     case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1780     case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1781     case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1782     case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1783     default:
1784       ShouldNotReachHere();
1785     }
1786   } else if (left->is_double_fpu()) {
1787     if (right->is_double_fpu()) {
1788       // fpu register - fpu register
1789       switch (code) {
1790       case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1791       case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1792       case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1793       case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1794       default:
1795         ShouldNotReachHere();
1796       }
1797     } else {
1798       if (right->is_constant()) {
1799         ShouldNotReachHere();
1800       }
1801       ShouldNotReachHere();
1802     }
1803   } else if (left->is_single_stack() || left->is_address()) {
1804     assert(left == dest, "left and dest must be equal");
1805     ShouldNotReachHere();
1806   } else {
1807     ShouldNotReachHere();
1808   }
1809 }
1810 
1811 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1812 
1813 
1814 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1815   switch(code) {
1816   case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1817   case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1818   default      : ShouldNotReachHere();
1819   }
1820 }
1821 
1822 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1823 
1824   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1825   Register Rleft = left->is_single_cpu() ? left->as_register() :
1826                                            left->as_register_lo();
1827    if (dst->is_single_cpu()) {
1828      Register Rdst = dst->as_register();
1829      if (right->is_constant()) {
1830        switch (code) {
1831          case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1832          case lir_logic_or:  __ orrw (Rdst, Rleft, right->as_jint()); break;
1833          case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1834          default: ShouldNotReachHere(); break;
1835        }
1836      } else {
1837        Register Rright = right->is_single_cpu() ? right->as_register() :
1838                                                   right->as_register_lo();
1839        switch (code) {
1840          case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1841          case lir_logic_or:  __ orrw (Rdst, Rleft, Rright); break;
1842          case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1843          default: ShouldNotReachHere(); break;
1844        }
1845      }
1846    } else {
1847      Register Rdst = dst->as_register_lo();
1848      if (right->is_constant()) {
1849        switch (code) {
1850          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1851          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jlong()); break;
1852          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1853          default: ShouldNotReachHere(); break;
1854        }
1855      } else {
1856        Register Rright = right->is_single_cpu() ? right->as_register() :
1857                                                   right->as_register_lo();
1858        switch (code) {
1859          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1860          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
1861          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1862          default: ShouldNotReachHere(); break;
1863        }
1864      }
1865    }
1866 }
1867 
1868 
1869 
1870 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) {
1871 
1872   // opcode check
1873   assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem");
1874   bool is_irem = (code == lir_irem);
1875 
1876   // operand check
1877   assert(left->is_single_cpu(),   "left must be register");
1878   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
1879   assert(result->is_single_cpu(), "result must be register");
1880   Register lreg = left->as_register();
1881   Register dreg = result->as_register();
1882 
1883   // power-of-2 constant check and codegen
1884   if (right->is_constant()) {
1885     int c = right->as_constant_ptr()->as_jint();
1886     assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1887     if (is_irem) {
1888       if (c == 1) {
1889         // move 0 to dreg if divisor is 1
1890         __ movw(dreg, zr);
1891       } else {
1892         // use rscratch1 as intermediate result register
1893         __ negsw(rscratch1, lreg);
1894         __ andw(dreg, lreg, c - 1);
1895         __ andw(rscratch1, rscratch1, c - 1);
1896         __ csnegw(dreg, dreg, rscratch1, Assembler::MI);
1897       }
1898     } else {
1899       if (c == 1) {
1900         // move lreg to dreg if divisor is 1
1901         __ movw(dreg, lreg);
1902       } else {
1903         unsigned int shift = exact_log2(c);
1904         // use rscratch1 as intermediate result register
1905         __ asrw(rscratch1, lreg, 31);
1906         __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift);
1907         __ asrw(dreg, rscratch1, shift);
1908       }
1909     }
1910   } else {
1911     Register rreg = right->as_register();
1912     __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1);
1913   }
1914 }
1915 
1916 
1917 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1918   if (opr1->is_constant() && opr2->is_single_cpu()) {
1919     // tableswitch
1920     Register reg = as_reg(opr2);
1921     struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1922     __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1923   } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1924     Register reg1 = as_reg(opr1);
1925     if (opr2->is_single_cpu()) {
1926       // cpu register - cpu register
1927       Register reg2 = opr2->as_register();
1928       if (is_reference_type(opr1->type())) {
1929         __ cmpoop(reg1, reg2);
1930       } else {
1931         assert(!is_reference_type(opr2->type()), "cmp int, oop?");
1932         __ cmpw(reg1, reg2);
1933       }
1934       return;
1935     }
1936     if (opr2->is_double_cpu()) {
1937       // cpu register - cpu register
1938       Register reg2 = opr2->as_register_lo();
1939       __ cmp(reg1, reg2);
1940       return;
1941     }
1942 
1943     if (opr2->is_constant()) {
1944       bool is_32bit = false; // width of register operand
1945       jlong imm;
1946 
1947       switch(opr2->type()) {
1948       case T_INT:
1949         imm = opr2->as_constant_ptr()->as_jint();
1950         is_32bit = true;
1951         break;
1952       case T_LONG:
1953         imm = opr2->as_constant_ptr()->as_jlong();
1954         break;
1955       case T_ADDRESS:
1956         imm = opr2->as_constant_ptr()->as_jint();
1957         break;
1958       case T_METADATA:
1959         imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
1960         break;
1961       case T_OBJECT:
1962       case T_ARRAY:
1963         jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
1964         __ cmpoop(reg1, rscratch1);
1965         return;
1966       default:
1967         ShouldNotReachHere();
1968         imm = 0;  // unreachable
1969         break;
1970       }
1971 
1972       if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1973         if (is_32bit)
1974           __ cmpw(reg1, imm);
1975         else
1976           __ subs(zr, reg1, imm);
1977         return;
1978       } else {
1979         __ mov(rscratch1, imm);
1980         if (is_32bit)
1981           __ cmpw(reg1, rscratch1);
1982         else
1983           __ cmp(reg1, rscratch1);
1984         return;
1985       }
1986     } else
1987       ShouldNotReachHere();
1988   } else if (opr1->is_single_fpu()) {
1989     FloatRegister reg1 = opr1->as_float_reg();
1990     assert(opr2->is_single_fpu(), "expect single float register");
1991     FloatRegister reg2 = opr2->as_float_reg();
1992     __ fcmps(reg1, reg2);
1993   } else if (opr1->is_double_fpu()) {
1994     FloatRegister reg1 = opr1->as_double_reg();
1995     assert(opr2->is_double_fpu(), "expect double float register");
1996     FloatRegister reg2 = opr2->as_double_reg();
1997     __ fcmpd(reg1, reg2);
1998   } else {
1999     ShouldNotReachHere();
2000   }
2001 }
2002 
2003 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2004   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2005     bool is_unordered_less = (code == lir_ucmp_fd2i);
2006     if (left->is_single_fpu()) {
2007       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2008     } else if (left->is_double_fpu()) {
2009       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2010     } else {
2011       ShouldNotReachHere();
2012     }
2013   } else if (code == lir_cmp_l2i) {
2014     Label done;
2015     __ cmp(left->as_register_lo(), right->as_register_lo());
2016     __ mov(dst->as_register(), (uint64_t)-1L);
2017     __ br(Assembler::LT, done);
2018     __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2019     __ bind(done);
2020   } else {
2021     ShouldNotReachHere();
2022   }
2023 }
2024 
2025 
2026 void LIR_Assembler::align_call(LIR_Code code) {  }
2027 
2028 
2029 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2030   address call = __ trampoline_call(Address(op->addr(), rtype));
2031   if (call == NULL) {
2032     bailout("trampoline stub overflow");
2033     return;
2034   }
2035   add_call_info(code_offset(), op->info());
2036   __ post_call_nop();
2037 }
2038 
2039 
2040 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2041   address call = __ ic_call(op->addr());
2042   if (call == NULL) {
2043     bailout("trampoline stub overflow");
2044     return;
2045   }
2046   add_call_info(code_offset(), op->info());
2047   __ post_call_nop();
2048 }
2049 
2050 void LIR_Assembler::emit_static_call_stub() {
2051   address call_pc = __ pc();
2052   address stub = __ start_a_stub(call_stub_size());
2053   if (stub == NULL) {
2054     bailout("static call stub overflow");
2055     return;
2056   }
2057 
2058   int start = __ offset();
2059 
2060   __ relocate(static_stub_Relocation::spec(call_pc));
2061   __ emit_static_call_stub();
2062 
2063   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
2064         <= call_stub_size(), "stub too big");
2065   __ end_a_stub();
2066 }
2067 
2068 
2069 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2070   assert(exceptionOop->as_register() == r0, "must match");
2071   assert(exceptionPC->as_register() == r3, "must match");
2072 
2073   // exception object is not added to oop map by LinearScan
2074   // (LinearScan assumes that no oops are in fixed registers)
2075   info->add_register_oop(exceptionOop);
2076   Runtime1::StubID unwind_id;
2077 
2078   // get current pc information
2079   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2080   if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
2081     // As no instructions have been generated yet for this LIR node it's
2082     // possible that an oop map already exists for the current offset.
2083     // In that case insert an dummy NOP here to ensure all oop map PCs
2084     // are unique. See JDK-8237483.
2085     __ nop();
2086   }
2087   int pc_for_athrow_offset = __ offset();
2088   InternalAddress pc_for_athrow(__ pc());
2089   __ adr(exceptionPC->as_register(), pc_for_athrow);
2090   add_call_info(pc_for_athrow_offset, info); // for exception handler
2091 
2092   __ verify_not_null_oop(r0);
2093   // search an exception handler (r0: exception oop, r3: throwing pc)
2094   if (compilation()->has_fpu_code()) {
2095     unwind_id = Runtime1::handle_exception_id;
2096   } else {
2097     unwind_id = Runtime1::handle_exception_nofpu_id;
2098   }
2099   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2100 
2101   // FIXME: enough room for two byte trap   ????
2102   __ nop();
2103 }
2104 
2105 
2106 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2107   assert(exceptionOop->as_register() == r0, "must match");
2108 
2109   __ b(_unwind_handler_entry);
2110 }
2111 
2112 
2113 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2114   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2115   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2116 
2117   switch (left->type()) {
2118     case T_INT: {
2119       switch (code) {
2120       case lir_shl:  __ lslvw (dreg, lreg, count->as_register()); break;
2121       case lir_shr:  __ asrvw (dreg, lreg, count->as_register()); break;
2122       case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2123       default:
2124         ShouldNotReachHere();
2125         break;
2126       }
2127       break;
2128     case T_LONG:
2129     case T_ADDRESS:
2130     case T_OBJECT:
2131       switch (code) {
2132       case lir_shl:  __ lslv (dreg, lreg, count->as_register()); break;
2133       case lir_shr:  __ asrv (dreg, lreg, count->as_register()); break;
2134       case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2135       default:
2136         ShouldNotReachHere();
2137         break;
2138       }
2139       break;
2140     default:
2141       ShouldNotReachHere();
2142       break;
2143     }
2144   }
2145 }
2146 
2147 
2148 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2149   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2150   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2151 
2152   switch (left->type()) {
2153     case T_INT: {
2154       switch (code) {
2155       case lir_shl:  __ lslw (dreg, lreg, count); break;
2156       case lir_shr:  __ asrw (dreg, lreg, count); break;
2157       case lir_ushr: __ lsrw (dreg, lreg, count); break;
2158       default:
2159         ShouldNotReachHere();
2160         break;
2161       }
2162       break;
2163     case T_LONG:
2164     case T_ADDRESS:
2165     case T_OBJECT:
2166       switch (code) {
2167       case lir_shl:  __ lsl (dreg, lreg, count); break;
2168       case lir_shr:  __ asr (dreg, lreg, count); break;
2169       case lir_ushr: __ lsr (dreg, lreg, count); break;
2170       default:
2171         ShouldNotReachHere();
2172         break;
2173       }
2174       break;
2175     default:
2176       ShouldNotReachHere();
2177       break;
2178     }
2179   }
2180 }
2181 
2182 
2183 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2184   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2185   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2186   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2187   __ str (r, Address(sp, offset_from_rsp_in_bytes));
2188 }
2189 
2190 
2191 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2192   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2193   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2194   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2195   __ mov (rscratch1, c);
2196   __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2197 }
2198 
2199 
2200 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2201   ShouldNotReachHere();
2202   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2203   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2204   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2205   __ lea(rscratch1, __ constant_oop_address(o));
2206   __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2207 }
2208 
2209 
2210 // This code replaces a call to arraycopy; no exception may
2211 // be thrown in this code, they must be thrown in the System.arraycopy
2212 // activation frame; we could save some checks if this would not be the case
2213 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2214   ciArrayKlass* default_type = op->expected_type();
2215   Register src = op->src()->as_register();
2216   Register dst = op->dst()->as_register();
2217   Register src_pos = op->src_pos()->as_register();
2218   Register dst_pos = op->dst_pos()->as_register();
2219   Register length  = op->length()->as_register();
2220   Register tmp = op->tmp()->as_register();
2221 
2222   CodeStub* stub = op->stub();
2223   int flags = op->flags();
2224   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2225   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2226 
2227   // if we don't know anything, just go through the generic arraycopy
2228   if (default_type == NULL // || basic_type == T_OBJECT
2229       ) {
2230     Label done;
2231     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2232 
2233     // Save the arguments in case the generic arraycopy fails and we
2234     // have to fall back to the JNI stub
2235     __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2236     __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2237     __ str(src,              Address(sp, 4*BytesPerWord));
2238 
2239     address copyfunc_addr = StubRoutines::generic_arraycopy();
2240     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2241 
2242     // The arguments are in java calling convention so we shift them
2243     // to C convention
2244     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2245     __ mov(c_rarg0, j_rarg0);
2246     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2247     __ mov(c_rarg1, j_rarg1);
2248     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2249     __ mov(c_rarg2, j_rarg2);
2250     assert_different_registers(c_rarg3, j_rarg4);
2251     __ mov(c_rarg3, j_rarg3);
2252     __ mov(c_rarg4, j_rarg4);
2253 #ifndef PRODUCT
2254     if (PrintC1Statistics) {
2255       __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2256     }
2257 #endif
2258     __ far_call(RuntimeAddress(copyfunc_addr));
2259 
2260     __ cbz(r0, *stub->continuation());
2261 
2262     // Reload values from the stack so they are where the stub
2263     // expects them.
2264     __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2265     __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2266     __ ldr(src,              Address(sp, 4*BytesPerWord));
2267 
2268     // r0 is -1^K where K == partial copied count
2269     __ eonw(rscratch1, r0, zr);
2270     // adjust length down and src/end pos up by partial copied count
2271     __ subw(length, length, rscratch1);
2272     __ addw(src_pos, src_pos, rscratch1);
2273     __ addw(dst_pos, dst_pos, rscratch1);
2274     __ b(*stub->entry());
2275 
2276     __ bind(*stub->continuation());
2277     return;
2278   }
2279 
2280   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2281 
2282   int elem_size = type2aelembytes(basic_type);
2283   int scale = exact_log2(elem_size);
2284 
2285   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2286   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2287 
2288   // test for NULL
2289   if (flags & LIR_OpArrayCopy::src_null_check) {
2290     __ cbz(src, *stub->entry());
2291   }
2292   if (flags & LIR_OpArrayCopy::dst_null_check) {
2293     __ cbz(dst, *stub->entry());
2294   }
2295 
2296   // If the compiler was not able to prove that exact type of the source or the destination
2297   // of the arraycopy is an array type, check at runtime if the source or the destination is
2298   // an instance type.
2299   if (flags & LIR_OpArrayCopy::type_check) {
2300     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2301       __ load_klass(tmp, dst);
2302       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2303       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2304       __ br(Assembler::GE, *stub->entry());
2305     }
2306 
2307     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2308       __ load_klass(tmp, src);
2309       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2310       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2311       __ br(Assembler::GE, *stub->entry());
2312     }
2313   }
2314 
2315   // check if negative
2316   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2317     __ cmpw(src_pos, 0);
2318     __ br(Assembler::LT, *stub->entry());
2319   }
2320   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2321     __ cmpw(dst_pos, 0);
2322     __ br(Assembler::LT, *stub->entry());
2323   }
2324 
2325   if (flags & LIR_OpArrayCopy::length_positive_check) {
2326     __ cmpw(length, 0);
2327     __ br(Assembler::LT, *stub->entry());
2328   }
2329 
2330   if (flags & LIR_OpArrayCopy::src_range_check) {
2331     __ addw(tmp, src_pos, length);
2332     __ ldrw(rscratch1, src_length_addr);
2333     __ cmpw(tmp, rscratch1);
2334     __ br(Assembler::HI, *stub->entry());
2335   }
2336   if (flags & LIR_OpArrayCopy::dst_range_check) {
2337     __ addw(tmp, dst_pos, length);
2338     __ ldrw(rscratch1, dst_length_addr);
2339     __ cmpw(tmp, rscratch1);
2340     __ br(Assembler::HI, *stub->entry());
2341   }
2342 
2343   if (flags & LIR_OpArrayCopy::type_check) {
2344     // We don't know the array types are compatible
2345     if (basic_type != T_OBJECT) {
2346       // Simple test for basic type arrays
2347       assert(UseCompressedClassPointers, "Lilliput");
2348       __ load_nklass(tmp, src);
2349       __ load_nklass(rscratch1, dst);
2350       __ cmpw(tmp, rscratch1);
2351       __ br(Assembler::NE, *stub->entry());
2352     } else {
2353       // For object arrays, if src is a sub class of dst then we can
2354       // safely do the copy.
2355       Label cont, slow;
2356 
2357 #define PUSH(r1, r2)                                    \
2358       stp(r1, r2, __ pre(sp, -2 * wordSize));
2359 
2360 #define POP(r1, r2)                                     \
2361       ldp(r1, r2, __ post(sp, 2 * wordSize));
2362 
2363       __ PUSH(src, dst);
2364 
2365       __ load_klass(tmp, src);
2366       __ mov(src, tmp);
2367       __ load_klass(tmp, dst);
2368       __ mov(dst, tmp);
2369 
2370       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2371 
2372       __ PUSH(src, dst);
2373       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2374       __ POP(src, dst);
2375 
2376       __ cbnz(src, cont);
2377 
2378       __ bind(slow);
2379       __ POP(src, dst);
2380 
2381       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2382       if (copyfunc_addr != NULL) { // use stub if available
2383         // src is not a sub class of dst so we have to do a
2384         // per-element check.
2385 
2386         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2387         if ((flags & mask) != mask) {
2388           // Check that at least both of them object arrays.
2389           assert(flags & mask, "one of the two should be known to be an object array");
2390 
2391           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2392             __ load_klass(tmp, src);
2393           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2394             __ load_klass(tmp, dst);
2395           }
2396           int lh_offset = in_bytes(Klass::layout_helper_offset());
2397           Address klass_lh_addr(tmp, lh_offset);
2398           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2399           __ ldrw(rscratch1, klass_lh_addr);
2400           __ mov(rscratch2, objArray_lh);
2401           __ eorw(rscratch1, rscratch1, rscratch2);
2402           __ cbnzw(rscratch1, *stub->entry());
2403         }
2404 
2405        // Spill because stubs can use any register they like and it's
2406        // easier to restore just those that we care about.
2407         __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2408         __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2409         __ str(src,              Address(sp, 4*BytesPerWord));
2410 
2411         __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2412         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2413         assert_different_registers(c_rarg0, dst, dst_pos, length);
2414         __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2415         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2416         assert_different_registers(c_rarg1, dst, length);
2417         __ uxtw(c_rarg2, length);
2418         assert_different_registers(c_rarg2, dst);
2419 
2420         __ load_klass(c_rarg4, dst);
2421         __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2422         __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2423         __ far_call(RuntimeAddress(copyfunc_addr));
2424 
2425 #ifndef PRODUCT
2426         if (PrintC1Statistics) {
2427           Label failed;
2428           __ cbnz(r0, failed);
2429           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2430           __ bind(failed);
2431         }
2432 #endif
2433 
2434         __ cbz(r0, *stub->continuation());
2435 
2436 #ifndef PRODUCT
2437         if (PrintC1Statistics) {
2438           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2439         }
2440 #endif
2441         assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2442 
2443         // Restore previously spilled arguments
2444         __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2445         __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2446         __ ldr(src,              Address(sp, 4*BytesPerWord));
2447 
2448         // return value is -1^K where K is partial copied count
2449         __ eonw(rscratch1, r0, zr);
2450         // adjust length down and src/end pos up by partial copied count
2451         __ subw(length, length, rscratch1);
2452         __ addw(src_pos, src_pos, rscratch1);
2453         __ addw(dst_pos, dst_pos, rscratch1);
2454       }
2455 
2456       __ b(*stub->entry());
2457 
2458       __ bind(cont);
2459       __ POP(src, dst);
2460     }
2461   }
2462 
2463 #ifdef ASSERT
2464   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2465     // Sanity check the known type with the incoming class.  For the
2466     // primitive case the types must match exactly with src.klass and
2467     // dst.klass each exactly matching the default type.  For the
2468     // object array case, if no type check is needed then either the
2469     // dst type is exactly the expected type and the src type is a
2470     // subtype which we can't check or src is the same array as dst
2471     // but not necessarily exactly of type default_type.
2472     Label known_ok, halt;
2473     __ mov_metadata(tmp, default_type->constant_encoding());
2474     if (UseCompressedClassPointers) {
2475       __ encode_klass_not_null(tmp);
2476     }
2477 
2478     assert(UseCompressedClassPointers, "Lilliput");
2479     if (basic_type != T_OBJECT) {
2480       __ load_nklass(rscratch1, dst);
2481       __ cmpw(tmp, rscratch1);
2482       __ br(Assembler::NE, halt);
2483       __ load_nklass(rscratch1, src);
2484       __ cmpw(tmp, rscratch1);
2485       __ br(Assembler::EQ, known_ok);
2486     } else {
2487       __ load_nklass(rscratch1, dst);
2488       __ cmpw(tmp, rscratch1);
2489       __ br(Assembler::EQ, known_ok);
2490       __ cmp(src, dst);
2491       __ br(Assembler::EQ, known_ok);
2492     }
2493     __ bind(halt);
2494     __ stop("incorrect type information in arraycopy");
2495     __ bind(known_ok);
2496   }
2497 #endif
2498 
2499 #ifndef PRODUCT
2500   if (PrintC1Statistics) {
2501     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2502   }
2503 #endif
2504 
2505   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2506   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2507   assert_different_registers(c_rarg0, dst, dst_pos, length);
2508   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2509   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2510   assert_different_registers(c_rarg1, dst, length);
2511   __ uxtw(c_rarg2, length);
2512   assert_different_registers(c_rarg2, dst);
2513 
2514   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2515   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2516   const char *name;
2517   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2518 
2519  CodeBlob *cb = CodeCache::find_blob(entry);
2520  if (cb) {
2521    __ far_call(RuntimeAddress(entry));
2522  } else {
2523    __ call_VM_leaf(entry, 3);
2524  }
2525 
2526   __ bind(*stub->continuation());
2527 }
2528 
2529 
2530 
2531 
2532 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2533   Register obj = op->obj_opr()->as_register();  // may not be an oop
2534   Register hdr = op->hdr_opr()->as_register();
2535   Register lock = op->lock_opr()->as_register();
2536   if (UseHeavyMonitors) {
2537     if (op->info() != NULL) {
2538       add_debug_info_for_null_check_here(op->info());
2539       __ null_check(obj, -1);
2540     }
2541     __ b(*op->stub()->entry());
2542   } else if (op->code() == lir_lock) {
2543     // add debug info for NullPointerException only if one is possible
2544     int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry());
2545     if (op->info() != NULL) {
2546       add_debug_info_for_null_check(null_check_offset, op->info());
2547     }
2548     // done
2549   } else if (op->code() == lir_unlock) {
2550     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2551   } else {
2552     Unimplemented();
2553   }
2554   __ bind(*op->stub()->continuation());
2555 }
2556 
2557 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2558   Register obj = op->obj()->as_pointer_register();
2559   Register result = op->result_opr()->as_pointer_register();
2560   Register tmp = rscratch1;
2561 
2562   CodeEmitInfo* info = op->info();
2563   if (info != NULL) {
2564     add_debug_info_for_null_check_here(info);
2565   }
2566 
2567   assert(UseCompressedClassPointers, "expects UseCompressedClassPointers");
2568 
2569   // Check if we can take the (common) fast path, if obj is unlocked.
2570   __ ldr(tmp, Address(obj, oopDesc::mark_offset_in_bytes()));
2571   __ eor(tmp, tmp, markWord::unlocked_value);
2572   __ tst(tmp, markWord::lock_mask_in_place);
2573   __ br(Assembler::NE, *op->stub()->entry());
2574 
2575   // Fast-path: shift and decode Klass*.
2576   __ mov(result, tmp);
2577   __ lsr(result, result, markWord::klass_shift);
2578 
2579   __ bind(*op->stub()->continuation());
2580   __ decode_klass_not_null(result);
2581 }
2582 
2583 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2584   ciMethod* method = op->profiled_method();
2585   int bci          = op->profiled_bci();
2586   ciMethod* callee = op->profiled_callee();
2587 
2588   // Update counter for all call types
2589   ciMethodData* md = method->method_data_or_null();
2590   assert(md != NULL, "Sanity");
2591   ciProfileData* data = md->bci_to_data(bci);
2592   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2593   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2594   Register mdo  = op->mdo()->as_register();
2595   __ mov_metadata(mdo, md->constant_encoding());
2596   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2597   // Perform additional virtual call profiling for invokevirtual and
2598   // invokeinterface bytecodes
2599   if (op->should_profile_receiver_type()) {
2600     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2601     Register recv = op->recv()->as_register();
2602     assert_different_registers(mdo, recv);
2603     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2604     ciKlass* known_klass = op->known_holder();
2605     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2606       // We know the type that will be seen at this call site; we can
2607       // statically update the MethodData* rather than needing to do
2608       // dynamic tests on the receiver type
2609 
2610       // NOTE: we should probably put a lock around this search to
2611       // avoid collisions by concurrent compilations
2612       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2613       uint i;
2614       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2615         ciKlass* receiver = vc_data->receiver(i);
2616         if (known_klass->equals(receiver)) {
2617           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2618           __ addptr(data_addr, DataLayout::counter_increment);
2619           return;
2620         }
2621       }
2622 
2623       // Receiver type not found in profile data; select an empty slot
2624 
2625       // Note that this is less efficient than it should be because it
2626       // always does a write to the receiver part of the
2627       // VirtualCallData rather than just the first time
2628       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2629         ciKlass* receiver = vc_data->receiver(i);
2630         if (receiver == NULL) {
2631           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2632           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2633           __ lea(rscratch2, recv_addr);
2634           __ str(rscratch1, Address(rscratch2));
2635           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2636           __ addptr(data_addr, DataLayout::counter_increment);
2637           return;
2638         }
2639       }
2640     } else {
2641       __ load_klass(rscratch1, recv);
2642       __ mov(recv, rscratch1);
2643       Label update_done;
2644       type_profile_helper(mdo, md, data, recv, &update_done);
2645       // Receiver did not match any saved receiver and there is no empty row for it.
2646       // Increment total counter to indicate polymorphic case.
2647       __ addptr(counter_addr, DataLayout::counter_increment);
2648 
2649       __ bind(update_done);
2650     }
2651   } else {
2652     // Static call
2653     __ addptr(counter_addr, DataLayout::counter_increment);
2654   }
2655 }
2656 
2657 
2658 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2659   Unimplemented();
2660 }
2661 
2662 
2663 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2664   __ lea(dst->as_register(), frame_map()->address_for_monitor_object(monitor_no));
2665 }
2666 
2667 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2668   assert(op->crc()->is_single_cpu(),  "crc must be register");
2669   assert(op->val()->is_single_cpu(),  "byte value must be register");
2670   assert(op->result_opr()->is_single_cpu(), "result must be register");
2671   Register crc = op->crc()->as_register();
2672   Register val = op->val()->as_register();
2673   Register res = op->result_opr()->as_register();
2674 
2675   assert_different_registers(val, crc, res);
2676   uint64_t offset;
2677   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2678   __ add(res, res, offset);
2679 
2680   __ mvnw(crc, crc); // ~crc
2681   __ update_byte_crc32(crc, val, res);
2682   __ mvnw(res, crc); // ~crc
2683 }
2684 
2685 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2686   COMMENT("emit_profile_type {");
2687   Register obj = op->obj()->as_register();
2688   Register tmp = op->tmp()->as_pointer_register();
2689   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2690   ciKlass* exact_klass = op->exact_klass();
2691   intptr_t current_klass = op->current_klass();
2692   bool not_null = op->not_null();
2693   bool no_conflict = op->no_conflict();
2694 
2695   Label update, next, none;
2696 
2697   bool do_null = !not_null;
2698   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2699   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2700 
2701   assert(do_null || do_update, "why are we here?");
2702   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2703   assert(mdo_addr.base() != rscratch1, "wrong register");
2704 
2705   __ verify_oop(obj);
2706 
2707   if (tmp != obj) {
2708     __ mov(tmp, obj);
2709   }
2710   if (do_null) {
2711     __ cbnz(tmp, update);
2712     if (!TypeEntries::was_null_seen(current_klass)) {
2713       __ ldr(rscratch2, mdo_addr);
2714       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2715       __ str(rscratch2, mdo_addr);
2716     }
2717     if (do_update) {
2718 #ifndef ASSERT
2719       __ b(next);
2720     }
2721 #else
2722       __ b(next);
2723     }
2724   } else {
2725     __ cbnz(tmp, update);
2726     __ stop("unexpected null obj");
2727 #endif
2728   }
2729 
2730   __ bind(update);
2731 
2732   if (do_update) {
2733 #ifdef ASSERT
2734     if (exact_klass != NULL) {
2735       Label ok;
2736       __ load_klass(rscratch1, tmp);
2737       __ mov(tmp, rscratch1);
2738       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2739       __ eor(rscratch1, tmp, rscratch1);
2740       __ cbz(rscratch1, ok);
2741       __ stop("exact klass and actual klass differ");
2742       __ bind(ok);
2743     }
2744 #endif
2745     if (!no_conflict) {
2746       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2747         if (exact_klass != NULL) {
2748           __ mov_metadata(tmp, exact_klass->constant_encoding());
2749         } else {
2750           __ load_klass(rscratch1, tmp);
2751           __ mov(tmp, rscratch1);
2752         }
2753 
2754         __ ldr(rscratch2, mdo_addr);
2755         __ eor(tmp, tmp, rscratch2);
2756         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2757         // klass seen before, nothing to do. The unknown bit may have been
2758         // set already but no need to check.
2759         __ cbz(rscratch1, next);
2760 
2761         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2762 
2763         if (TypeEntries::is_type_none(current_klass)) {
2764           __ cbz(rscratch2, none);
2765           __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2766           __ br(Assembler::EQ, none);
2767           // There is a chance that the checks above (re-reading profiling
2768           // data from memory) fail if another thread has just set the
2769           // profiling to this obj's klass
2770           __ dmb(Assembler::ISHLD);
2771           __ ldr(rscratch2, mdo_addr);
2772           __ eor(tmp, tmp, rscratch2);
2773           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2774           __ cbz(rscratch1, next);
2775         }
2776       } else {
2777         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2778                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2779 
2780         __ ldr(tmp, mdo_addr);
2781         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2782       }
2783 
2784       // different than before. Cannot keep accurate profile.
2785       __ ldr(rscratch2, mdo_addr);
2786       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2787       __ str(rscratch2, mdo_addr);
2788 
2789       if (TypeEntries::is_type_none(current_klass)) {
2790         __ b(next);
2791 
2792         __ bind(none);
2793         // first time here. Set profile type.
2794         __ str(tmp, mdo_addr);
2795       }
2796     } else {
2797       // There's a single possible klass at this profile point
2798       assert(exact_klass != NULL, "should be");
2799       if (TypeEntries::is_type_none(current_klass)) {
2800         __ mov_metadata(tmp, exact_klass->constant_encoding());
2801         __ ldr(rscratch2, mdo_addr);
2802         __ eor(tmp, tmp, rscratch2);
2803         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2804         __ cbz(rscratch1, next);
2805 #ifdef ASSERT
2806         {
2807           Label ok;
2808           __ ldr(rscratch1, mdo_addr);
2809           __ cbz(rscratch1, ok);
2810           __ cmp(rscratch1, (u1)TypeEntries::null_seen);
2811           __ br(Assembler::EQ, ok);
2812           // may have been set by another thread
2813           __ dmb(Assembler::ISHLD);
2814           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2815           __ ldr(rscratch2, mdo_addr);
2816           __ eor(rscratch2, rscratch1, rscratch2);
2817           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2818           __ cbz(rscratch2, ok);
2819 
2820           __ stop("unexpected profiling mismatch");
2821           __ bind(ok);
2822         }
2823 #endif
2824         // first time here. Set profile type.
2825         __ str(tmp, mdo_addr);
2826       } else {
2827         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2828                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2829 
2830         __ ldr(tmp, mdo_addr);
2831         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2832 
2833         __ orr(tmp, tmp, TypeEntries::type_unknown);
2834         __ str(tmp, mdo_addr);
2835         // FIXME: Write barrier needed here?
2836       }
2837     }
2838 
2839     __ bind(next);
2840   }
2841   COMMENT("} emit_profile_type");
2842 }
2843 
2844 
2845 void LIR_Assembler::align_backward_branch_target() {
2846 }
2847 
2848 
2849 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2850   // tmp must be unused
2851   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2852 
2853   if (left->is_single_cpu()) {
2854     assert(dest->is_single_cpu(), "expect single result reg");
2855     __ negw(dest->as_register(), left->as_register());
2856   } else if (left->is_double_cpu()) {
2857     assert(dest->is_double_cpu(), "expect double result reg");
2858     __ neg(dest->as_register_lo(), left->as_register_lo());
2859   } else if (left->is_single_fpu()) {
2860     assert(dest->is_single_fpu(), "expect single float result reg");
2861     __ fnegs(dest->as_float_reg(), left->as_float_reg());
2862   } else {
2863     assert(left->is_double_fpu(), "expect double float operand reg");
2864     assert(dest->is_double_fpu(), "expect double float result reg");
2865     __ fnegd(dest->as_double_reg(), left->as_double_reg());
2866   }
2867 }
2868 
2869 
2870 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2871   if (patch_code != lir_patch_none) {
2872     deoptimize_trap(info);
2873     return;
2874   }
2875 
2876   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2877 }
2878 
2879 
2880 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2881   assert(!tmp->is_valid(), "don't need temporary");
2882 
2883   CodeBlob *cb = CodeCache::find_blob(dest);
2884   if (cb) {
2885     __ far_call(RuntimeAddress(dest));
2886   } else {
2887     __ mov(rscratch1, RuntimeAddress(dest));
2888     __ blr(rscratch1);
2889   }
2890 
2891   if (info != NULL) {
2892     add_call_info_here(info);
2893   }
2894   __ post_call_nop();
2895 }
2896 
2897 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2898   if (dest->is_address() || src->is_address()) {
2899     move_op(src, dest, type, lir_patch_none, info,
2900             /*pop_fpu_stack*/false, /*wide*/false);
2901   } else {
2902     ShouldNotReachHere();
2903   }
2904 }
2905 
2906 #ifdef ASSERT
2907 // emit run-time assertion
2908 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2909   assert(op->code() == lir_assert, "must be");
2910 
2911   if (op->in_opr1()->is_valid()) {
2912     assert(op->in_opr2()->is_valid(), "both operands must be valid");
2913     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2914   } else {
2915     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2916     assert(op->condition() == lir_cond_always, "no other conditions allowed");
2917   }
2918 
2919   Label ok;
2920   if (op->condition() != lir_cond_always) {
2921     Assembler::Condition acond = Assembler::AL;
2922     switch (op->condition()) {
2923       case lir_cond_equal:        acond = Assembler::EQ;  break;
2924       case lir_cond_notEqual:     acond = Assembler::NE;  break;
2925       case lir_cond_less:         acond = Assembler::LT;  break;
2926       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
2927       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
2928       case lir_cond_greater:      acond = Assembler::GT;  break;
2929       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
2930       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
2931       default:                    ShouldNotReachHere();
2932     }
2933     __ br(acond, ok);
2934   }
2935   if (op->halt()) {
2936     const char* str = __ code_string(op->msg());
2937     __ stop(str);
2938   } else {
2939     breakpoint();
2940   }
2941   __ bind(ok);
2942 }
2943 #endif
2944 
2945 #ifndef PRODUCT
2946 #define COMMENT(x)   do { __ block_comment(x); } while (0)
2947 #else
2948 #define COMMENT(x)
2949 #endif
2950 
2951 void LIR_Assembler::membar() {
2952   COMMENT("membar");
2953   __ membar(MacroAssembler::AnyAny);
2954 }
2955 
2956 void LIR_Assembler::membar_acquire() {
2957   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2958 }
2959 
2960 void LIR_Assembler::membar_release() {
2961   __ membar(Assembler::LoadStore|Assembler::StoreStore);
2962 }
2963 
2964 void LIR_Assembler::membar_loadload() {
2965   __ membar(Assembler::LoadLoad);
2966 }
2967 
2968 void LIR_Assembler::membar_storestore() {
2969   __ membar(MacroAssembler::StoreStore);
2970 }
2971 
2972 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2973 
2974 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2975 
2976 void LIR_Assembler::on_spin_wait() {
2977   __ spin_wait();
2978 }
2979 
2980 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2981   __ mov(result_reg->as_register(), rthread);
2982 }
2983 
2984 
2985 void LIR_Assembler::peephole(LIR_List *lir) {
2986 #if 0
2987   if (tableswitch_count >= max_tableswitches)
2988     return;
2989 
2990   /*
2991     This finite-state automaton recognizes sequences of compare-and-
2992     branch instructions.  We will turn them into a tableswitch.  You
2993     could argue that C1 really shouldn't be doing this sort of
2994     optimization, but without it the code is really horrible.
2995   */
2996 
2997   enum { start_s, cmp1_s, beq_s, cmp_s } state;
2998   int first_key, last_key = -2147483648;
2999   int next_key = 0;
3000   int start_insn = -1;
3001   int last_insn = -1;
3002   Register reg = noreg;
3003   LIR_Opr reg_opr;
3004   state = start_s;
3005 
3006   LIR_OpList* inst = lir->instructions_list();
3007   for (int i = 0; i < inst->length(); i++) {
3008     LIR_Op* op = inst->at(i);
3009     switch (state) {
3010     case start_s:
3011       first_key = -1;
3012       start_insn = i;
3013       switch (op->code()) {
3014       case lir_cmp:
3015         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3016         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3017         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3018             && opr2->is_constant()
3019             && opr2->type() == T_INT) {
3020           reg_opr = opr1;
3021           reg = opr1->as_register();
3022           first_key = opr2->as_constant_ptr()->as_jint();
3023           next_key = first_key + 1;
3024           state = cmp_s;
3025           goto next_state;
3026         }
3027         break;
3028       }
3029       break;
3030     case cmp_s:
3031       switch (op->code()) {
3032       case lir_branch:
3033         if (op->as_OpBranch()->cond() == lir_cond_equal) {
3034           state = beq_s;
3035           last_insn = i;
3036           goto next_state;
3037         }
3038       }
3039       state = start_s;
3040       break;
3041     case beq_s:
3042       switch (op->code()) {
3043       case lir_cmp: {
3044         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3045         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3046         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3047             && opr1->as_register() == reg
3048             && opr2->is_constant()
3049             && opr2->type() == T_INT
3050             && opr2->as_constant_ptr()->as_jint() == next_key) {
3051           last_key = next_key;
3052           next_key++;
3053           state = cmp_s;
3054           goto next_state;
3055         }
3056       }
3057       }
3058       last_key = next_key;
3059       state = start_s;
3060       break;
3061     default:
3062       assert(false, "impossible state");
3063     }
3064     if (state == start_s) {
3065       if (first_key < last_key - 5L && reg != noreg) {
3066         {
3067           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3068           //        reg->encoding(),
3069           //        start_insn, first_key, last_key);
3070           //   for (int i = 0; i < inst->length(); i++) {
3071           //     inst->at(i)->print();
3072           //     tty->print("\n");
3073           //   }
3074           //   tty->print("\n");
3075         }
3076 
3077         struct tableswitch *sw = &switches[tableswitch_count];
3078         sw->_insn_index = start_insn, sw->_first_key = first_key,
3079           sw->_last_key = last_key, sw->_reg = reg;
3080         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3081         {
3082           // Insert the new table of branches
3083           int offset = last_insn;
3084           for (int n = first_key; n < last_key; n++) {
3085             inst->insert_before
3086               (last_insn + 1,
3087                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3088                                 inst->at(offset)->as_OpBranch()->label()));
3089             offset -= 2, i++;
3090           }
3091         }
3092         // Delete all the old compare-and-branch instructions
3093         for (int n = first_key; n < last_key; n++) {
3094           inst->remove_at(start_insn);
3095           inst->remove_at(start_insn);
3096         }
3097         // Insert the tableswitch instruction
3098         inst->insert_before(start_insn,
3099                             new LIR_Op2(lir_cmp, lir_cond_always,
3100                                         LIR_OprFact::intConst(tableswitch_count),
3101                                         reg_opr));
3102         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3103         tableswitch_count++;
3104       }
3105       reg = noreg;
3106       last_key = -2147483648;
3107     }
3108   next_state:
3109     ;
3110   }
3111 #endif
3112 }
3113 
3114 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3115   Address addr = as_Address(src->as_address_ptr());
3116   BasicType type = src->type();
3117   bool is_oop = is_reference_type(type);
3118 
3119   void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3120   void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3121 
3122   switch(type) {
3123   case T_INT:
3124     xchg = &MacroAssembler::atomic_xchgalw;
3125     add = &MacroAssembler::atomic_addalw;
3126     break;
3127   case T_LONG:
3128     xchg = &MacroAssembler::atomic_xchgal;
3129     add = &MacroAssembler::atomic_addal;
3130     break;
3131   case T_OBJECT:
3132   case T_ARRAY:
3133     if (UseCompressedOops) {
3134       xchg = &MacroAssembler::atomic_xchgalw;
3135       add = &MacroAssembler::atomic_addalw;
3136     } else {
3137       xchg = &MacroAssembler::atomic_xchgal;
3138       add = &MacroAssembler::atomic_addal;
3139     }
3140     break;
3141   default:
3142     ShouldNotReachHere();
3143     xchg = &MacroAssembler::atomic_xchgal;
3144     add = &MacroAssembler::atomic_addal; // unreachable
3145   }
3146 
3147   switch (code) {
3148   case lir_xadd:
3149     {
3150       RegisterOrConstant inc;
3151       Register tmp = as_reg(tmp_op);
3152       Register dst = as_reg(dest);
3153       if (data->is_constant()) {
3154         inc = RegisterOrConstant(as_long(data));
3155         assert_different_registers(dst, addr.base(), tmp,
3156                                    rscratch1, rscratch2);
3157       } else {
3158         inc = RegisterOrConstant(as_reg(data));
3159         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3160                                    rscratch1, rscratch2);
3161       }
3162       __ lea(tmp, addr);
3163       (_masm->*add)(dst, inc, tmp);
3164       break;
3165     }
3166   case lir_xchg:
3167     {
3168       Register tmp = tmp_op->as_register();
3169       Register obj = as_reg(data);
3170       Register dst = as_reg(dest);
3171       if (is_oop && UseCompressedOops) {
3172         __ encode_heap_oop(rscratch2, obj);
3173         obj = rscratch2;
3174       }
3175       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3176       __ lea(tmp, addr);
3177       (_masm->*xchg)(dst, obj, tmp);
3178       if (is_oop && UseCompressedOops) {
3179         __ decode_heap_oop(dst);
3180       }
3181     }
3182     break;
3183   default:
3184     ShouldNotReachHere();
3185   }
3186   __ membar(__ AnyAny);
3187 }
3188 
3189 #undef __