1 /*
   2  * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Compilation.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_MacroAssembler.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArrayKlass.hpp"
  36 #include "ci/ciInstance.hpp"
  37 #include "ci/ciUtilities.hpp"
  38 #include "code/SCCache.hpp"
  39 #include "code/compiledIC.hpp"
  40 #include "gc/shared/collectedHeap.hpp"
  41 #include "gc/shared/gc_globals.hpp"
  42 #include "nativeInst_aarch64.hpp"
  43 #include "oops/objArrayKlass.hpp"
  44 #include "runtime/frame.inline.hpp"
  45 #include "runtime/sharedRuntime.hpp"
  46 #include "runtime/stubRoutines.hpp"
  47 #include "utilities/powerOfTwo.hpp"
  48 #include "vmreg_aarch64.inline.hpp"
  49 
  50 
  51 #ifndef PRODUCT
  52 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  53 #else
  54 #define COMMENT(x)
  55 #endif
  56 
  57 NEEDS_CLEANUP // remove this definitions ?
  58 const Register SYNC_header = r0;   // synchronization header
  59 const Register SHIFT_count = r0;   // where count for shift operations must be
  60 
  61 #define __ _masm->
  62 
  63 
  64 static void select_different_registers(Register preserve,
  65                                        Register extra,
  66                                        Register &tmp1,
  67                                        Register &tmp2) {
  68   if (tmp1 == preserve) {
  69     assert_different_registers(tmp1, tmp2, extra);
  70     tmp1 = extra;
  71   } else if (tmp2 == preserve) {
  72     assert_different_registers(tmp1, tmp2, extra);
  73     tmp2 = extra;
  74   }
  75   assert_different_registers(preserve, tmp1, tmp2);
  76 }
  77 
  78 
  79 
  80 static void select_different_registers(Register preserve,
  81                                        Register extra,
  82                                        Register &tmp1,
  83                                        Register &tmp2,
  84                                        Register &tmp3) {
  85   if (tmp1 == preserve) {
  86     assert_different_registers(tmp1, tmp2, tmp3, extra);
  87     tmp1 = extra;
  88   } else if (tmp2 == preserve) {
  89     assert_different_registers(tmp1, tmp2, tmp3, extra);
  90     tmp2 = extra;
  91   } else if (tmp3 == preserve) {
  92     assert_different_registers(tmp1, tmp2, tmp3, extra);
  93     tmp3 = extra;
  94   }
  95   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  96 }
  97 
  98 
  99 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
 100 
 101 
 102 LIR_Opr LIR_Assembler::receiverOpr() {
 103   return FrameMap::receiver_opr;
 104 }
 105 
 106 LIR_Opr LIR_Assembler::osrBufferPointer() {
 107   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 108 }
 109 
 110 //--------------fpu register translations-----------------------
 111 
 112 
 113 address LIR_Assembler::float_constant(float f) {
 114   address const_addr = __ float_constant(f);
 115   if (const_addr == nullptr) {
 116     bailout("const section overflow");
 117     return __ code()->consts()->start();
 118   } else {
 119     return const_addr;
 120   }
 121 }
 122 
 123 
 124 address LIR_Assembler::double_constant(double d) {
 125   address const_addr = __ double_constant(d);
 126   if (const_addr == nullptr) {
 127     bailout("const section overflow");
 128     return __ code()->consts()->start();
 129   } else {
 130     return const_addr;
 131   }
 132 }
 133 
 134 address LIR_Assembler::int_constant(jlong n) {
 135   address const_addr = __ long_constant(n);
 136   if (const_addr == nullptr) {
 137     bailout("const section overflow");
 138     return __ code()->consts()->start();
 139   } else {
 140     return const_addr;
 141   }
 142 }
 143 
 144 void LIR_Assembler::breakpoint() { Unimplemented(); }
 145 
 146 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 147 
 148 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 149 
 150 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
 151 //-------------------------------------------
 152 
 153 static Register as_reg(LIR_Opr op) {
 154   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 155 }
 156 
 157 static jlong as_long(LIR_Opr data) {
 158   jlong result;
 159   switch (data->type()) {
 160   case T_INT:
 161     result = (data->as_jint());
 162     break;
 163   case T_LONG:
 164     result = (data->as_jlong());
 165     break;
 166   default:
 167     ShouldNotReachHere();
 168     result = 0;  // unreachable
 169   }
 170   return result;
 171 }
 172 
 173 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 174   Register base = addr->base()->as_pointer_register();
 175   LIR_Opr opr = addr->index();
 176   if (opr->is_cpu_register()) {
 177     Register index;
 178     if (opr->is_single_cpu())
 179       index = opr->as_register();
 180     else
 181       index = opr->as_register_lo();
 182     assert(addr->disp() == 0, "must be");
 183     switch(opr->type()) {
 184       case T_INT:
 185         return Address(base, index, Address::sxtw(addr->scale()));
 186       case T_LONG:
 187         return Address(base, index, Address::lsl(addr->scale()));
 188       default:
 189         ShouldNotReachHere();
 190       }
 191   } else {
 192     assert(addr->scale() == 0,
 193            "expected for immediate operand, was: %d", addr->scale());
 194     ptrdiff_t offset = ptrdiff_t(addr->disp());
 195     // NOTE: Does not handle any 16 byte vector access.
 196     const uint type_size = type2aelembytes(addr->type(), true);
 197     return __ legitimize_address(Address(base, offset), type_size, tmp);
 198   }
 199   return Address();
 200 }
 201 
 202 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 203   ShouldNotReachHere();
 204   return Address();
 205 }
 206 
 207 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 208   return as_Address(addr, rscratch1);
 209 }
 210 
 211 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 212   return as_Address(addr, rscratch1);  // Ouch
 213   // FIXME: This needs to be much more clever.  See x86.
 214 }
 215 
 216 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is
 217 // not encodable as a base + (immediate) offset, generate an explicit address
 218 // calculation to hold the address in a temporary register.
 219 Address LIR_Assembler::stack_slot_address(int index, uint size, Register tmp, int adjust) {
 220   precond(size == 4 || size == 8);
 221   Address addr = frame_map()->address_for_slot(index, adjust);
 222   precond(addr.getMode() == Address::base_plus_offset);
 223   precond(addr.base() == sp);
 224   precond(addr.offset() > 0);
 225   uint mask = size - 1;
 226   assert((addr.offset() & mask) == 0, "scaled offsets only");
 227   return __ legitimize_address(addr, size, tmp);
 228 }
 229 
 230 void LIR_Assembler::osr_entry() {
 231   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 232   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 233   ValueStack* entry_state = osr_entry->state();
 234   int number_of_locks = entry_state->locks_size();
 235 
 236   // we jump here if osr happens with the interpreter
 237   // state set up to continue at the beginning of the
 238   // loop that triggered osr - in particular, we have
 239   // the following registers setup:
 240   //
 241   // r2: osr buffer
 242   //
 243 
 244   // build frame
 245   ciMethod* m = compilation()->method();
 246   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 247 
 248   // OSR buffer is
 249   //
 250   // locals[nlocals-1..0]
 251   // monitors[0..number_of_locks]
 252   //
 253   // locals is a direct copy of the interpreter frame so in the osr buffer
 254   // so first slot in the local array is the last local from the interpreter
 255   // and last slot is local[0] (receiver) from the interpreter
 256   //
 257   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 258   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 259   // in the interpreter frame (the method lock if a sync method)
 260 
 261   // Initialize monitors in the compiled activation.
 262   //   r2: pointer to osr buffer
 263   //
 264   // All other registers are dead at this point and the locals will be
 265   // copied into place by code emitted in the IR.
 266 
 267   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 268   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 269     int monitor_offset = BytesPerWord * method()->max_locals() +
 270       (2 * BytesPerWord) * (number_of_locks - 1);
 271     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 272     // the OSR buffer using 2 word entries: first the lock and then
 273     // the oop.
 274     for (int i = 0; i < number_of_locks; i++) {
 275       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 276 #ifdef ASSERT
 277       // verify the interpreter's monitor has a non-null object
 278       {
 279         Label L;
 280         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 281         __ cbnz(rscratch1, L);
 282         __ stop("locked object is null");
 283         __ bind(L);
 284       }
 285 #endif
 286       __ ldr(r19, Address(OSR_buf, slot_offset));
 287       __ ldr(r20, Address(OSR_buf, slot_offset + BytesPerWord));
 288       __ str(r19, frame_map()->address_for_monitor_lock(i));
 289       __ str(r20, frame_map()->address_for_monitor_object(i));
 290     }
 291   }
 292 }
 293 
 294 
 295 // inline cache check; done before the frame is built.
 296 int LIR_Assembler::check_icache() {
 297   return __ ic_check(CodeEntryAlignment);
 298 }
 299 
 300 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 301   assert(VM_Version::supports_fast_class_init_checks(), "sanity");
 302   assert(!method->holder()->is_not_initialized(), "initialization should have been started");
 303 
 304   Label L_skip_barrier;
 305 
 306   __ mov_metadata(rscratch2, method->holder()->constant_encoding());
 307   __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier /*L_fast_path*/);
 308   __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 309   __ bind(L_skip_barrier);
 310 }
 311 
 312 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 313   if (o == nullptr) {
 314     __ mov(reg, zr);
 315   } else {
 316     __ movoop(reg, o);
 317   }
 318 }
 319 
 320 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 321   address target = nullptr;
 322   relocInfo::relocType reloc_type = relocInfo::none;
 323 
 324   switch (patching_id(info)) {
 325   case PatchingStub::access_field_id:
 326     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 327     reloc_type = relocInfo::section_word_type;
 328     break;
 329   case PatchingStub::load_klass_id:
 330     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 331     reloc_type = relocInfo::metadata_type;
 332     break;
 333   case PatchingStub::load_mirror_id:
 334     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 335     reloc_type = relocInfo::oop_type;
 336     break;
 337   case PatchingStub::load_appendix_id:
 338     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 339     reloc_type = relocInfo::oop_type;
 340     break;
 341   default: ShouldNotReachHere();
 342   }
 343 
 344   __ far_call(RuntimeAddress(target));
 345   add_call_info_here(info);
 346 }
 347 
 348 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 349   deoptimize_trap(info);
 350 }
 351 
 352 
 353 // This specifies the rsp decrement needed to build the frame
 354 int LIR_Assembler::initial_frame_size_in_bytes() const {
 355   // if rounding, must let FrameMap know!
 356 
 357   return in_bytes(frame_map()->framesize_in_bytes());
 358 }
 359 
 360 
 361 int LIR_Assembler::emit_exception_handler() {
 362   // generate code for exception handler
 363   address handler_base = __ start_a_stub(exception_handler_size());
 364   if (handler_base == nullptr) {
 365     // not enough space left for the handler
 366     bailout("exception handler overflow");
 367     return -1;
 368   }
 369 
 370   int offset = code_offset();
 371 
 372   // the exception oop and pc are in r0, and r3
 373   // no other registers need to be preserved, so invalidate them
 374   __ invalidate_registers(false, true, true, false, true, true);
 375 
 376   // check that there is really an exception
 377   __ verify_not_null_oop(r0);
 378 
 379   // search an exception handler (r0: exception oop, r3: throwing pc)
 380   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 381   __ should_not_reach_here();
 382   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 383   __ end_a_stub();
 384 
 385   return offset;
 386 }
 387 
 388 
 389 // Emit the code to remove the frame from the stack in the exception
 390 // unwind path.
 391 int LIR_Assembler::emit_unwind_handler() {
 392 #ifndef PRODUCT
 393   if (CommentedAssembly) {
 394     _masm->block_comment("Unwind handler");
 395   }
 396 #endif
 397 
 398   int offset = code_offset();
 399 
 400   // Fetch the exception from TLS and clear out exception related thread state
 401   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 402   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 403   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 404 
 405   __ bind(_unwind_handler_entry);
 406   __ verify_not_null_oop(r0);
 407   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 408     __ mov(r19, r0);  // Preserve the exception
 409   }
 410 
 411   // Perform needed unlocking
 412   MonitorExitStub* stub = nullptr;
 413   if (method()->is_synchronized()) {
 414     monitor_address(0, FrameMap::r0_opr);
 415     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
 416     if (LockingMode == LM_MONITOR) {
 417       __ b(*stub->entry());
 418     } else {
 419       __ unlock_object(r5, r4, r0, r6, *stub->entry());
 420     }
 421     __ bind(*stub->continuation());
 422   }
 423 
 424   if (compilation()->env()->dtrace_method_probes()) {
 425     __ mov(c_rarg0, rthread);
 426     __ mov_metadata(c_rarg1, method()->constant_encoding());
 427     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 428   }
 429 
 430   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 431     __ mov(r0, r19);  // Restore the exception
 432   }
 433 
 434   // remove the activation and dispatch to the unwind handler
 435   __ block_comment("remove_frame and dispatch to the unwind handler");
 436   __ remove_frame(initial_frame_size_in_bytes());
 437   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 438 
 439   // Emit the slow path assembly
 440   if (stub != nullptr) {
 441     stub->emit_code(this);
 442   }
 443 
 444   return offset;
 445 }
 446 
 447 
 448 int LIR_Assembler::emit_deopt_handler() {
 449   // generate code for exception handler
 450   address handler_base = __ start_a_stub(deopt_handler_size());
 451   if (handler_base == nullptr) {
 452     // not enough space left for the handler
 453     bailout("deopt handler overflow");
 454     return -1;
 455   }
 456 
 457   int offset = code_offset();
 458 
 459   __ adr(lr, pc());
 460   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 461   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 462   __ end_a_stub();
 463 
 464   return offset;
 465 }
 466 
 467 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 468   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 469   int pc_offset = code_offset();
 470   flush_debug_info(pc_offset);
 471   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 472   if (info->exception_handlers() != nullptr) {
 473     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 474   }
 475 }
 476 
 477 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 478   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 479 
 480   // Pop the stack before the safepoint code
 481   __ remove_frame(initial_frame_size_in_bytes());
 482 
 483   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 484     __ reserved_stack_check();
 485   }
 486 
 487   code_stub->set_safepoint_offset(__ offset());
 488   __ relocate(relocInfo::poll_return_type);
 489   __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */);
 490   __ ret(lr);
 491 }
 492 
 493 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 494   guarantee(info != nullptr, "Shouldn't be null");
 495   __ get_polling_page(rscratch1, relocInfo::poll_type);
 496   add_debug_info_for_branch(info);  // This isn't just debug info:
 497                                     // it's the oop map
 498   __ read_polling_page(rscratch1, relocInfo::poll_type);
 499   return __ offset();
 500 }
 501 
 502 
 503 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 504   if (from_reg == r31_sp)
 505     from_reg = sp;
 506   if (to_reg == r31_sp)
 507     to_reg = sp;
 508   __ mov(to_reg, from_reg);
 509 }
 510 
 511 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 512 
 513 
 514 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 515   assert(src->is_constant(), "should not call otherwise");
 516   assert(dest->is_register(), "should not call otherwise");
 517   LIR_Const* c = src->as_constant_ptr();
 518 
 519   switch (c->type()) {
 520     case T_INT: {
 521       assert(patch_code == lir_patch_none, "no patching handled here");
 522       __ movw(dest->as_register(), c->as_jint());
 523       break;
 524     }
 525 
 526     case T_ADDRESS: {
 527       assert(patch_code == lir_patch_none, "no patching handled here");
 528       __ mov(dest->as_register(), c->as_jint());
 529       break;
 530     }
 531 
 532     case T_LONG: {
 533       assert(patch_code == lir_patch_none, "no patching handled here");
 534       if (SCCache::is_on_for_write()) {
 535         // SCA needs relocation info for card table base
 536         address b = c->as_pointer();
 537         if (is_card_table_address(b)) {
 538           __ lea(dest->as_register_lo(), ExternalAddress(b));
 539           break;
 540         }
 541       }
 542       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
 543       break;
 544     }
 545 
 546     case T_OBJECT: {
 547         if (patch_code == lir_patch_none) {
 548           jobject2reg(c->as_jobject(), dest->as_register());
 549         } else {
 550           jobject2reg_with_patching(dest->as_register(), info);
 551         }
 552       break;
 553     }
 554 
 555     case T_METADATA: {
 556       if (patch_code != lir_patch_none) {
 557         klass2reg_with_patching(dest->as_register(), info);
 558       } else {
 559         __ mov_metadata(dest->as_register(), c->as_metadata());
 560       }
 561       break;
 562     }
 563 
 564     case T_FLOAT: {
 565       if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 566         __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
 567       } else {
 568         __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 569         __ ldrs(dest->as_float_reg(), Address(rscratch1));
 570       }
 571       break;
 572     }
 573 
 574     case T_DOUBLE: {
 575       if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
 576         __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
 577       } else {
 578         __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 579         __ ldrd(dest->as_double_reg(), Address(rscratch1));
 580       }
 581       break;
 582     }
 583 
 584     default:
 585       ShouldNotReachHere();
 586   }
 587 }
 588 
 589 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 590   LIR_Const* c = src->as_constant_ptr();
 591   switch (c->type()) {
 592   case T_OBJECT:
 593     {
 594       if (! c->as_jobject())
 595         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 596       else {
 597         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, nullptr);
 598         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 599       }
 600     }
 601     break;
 602   case T_ADDRESS:
 603     {
 604       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, nullptr);
 605       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 606     }
 607   case T_INT:
 608   case T_FLOAT:
 609     {
 610       Register reg = zr;
 611       if (c->as_jint_bits() == 0)
 612         __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 613       else {
 614         __ movw(rscratch1, c->as_jint_bits());
 615         __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 616       }
 617     }
 618     break;
 619   case T_LONG:
 620   case T_DOUBLE:
 621     {
 622       Register reg = zr;
 623       if (c->as_jlong_bits() == 0)
 624         __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 625                                                  lo_word_offset_in_bytes));
 626       else {
 627         __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
 628         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 629                                                         lo_word_offset_in_bytes));
 630       }
 631     }
 632     break;
 633   default:
 634     ShouldNotReachHere();
 635   }
 636 }
 637 
 638 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 639   assert(src->is_constant(), "should not call otherwise");
 640   LIR_Const* c = src->as_constant_ptr();
 641   LIR_Address* to_addr = dest->as_address_ptr();
 642 
 643   void (Assembler::* insn)(Register Rt, const Address &adr);
 644 
 645   switch (type) {
 646   case T_ADDRESS:
 647     assert(c->as_jint() == 0, "should be");
 648     insn = &Assembler::str;
 649     break;
 650   case T_LONG:
 651     assert(c->as_jlong() == 0, "should be");
 652     insn = &Assembler::str;
 653     break;
 654   case T_INT:
 655     assert(c->as_jint() == 0, "should be");
 656     insn = &Assembler::strw;
 657     break;
 658   case T_OBJECT:
 659   case T_ARRAY:
 660     assert(c->as_jobject() == 0, "should be");
 661     if (UseCompressedOops && !wide) {
 662       insn = &Assembler::strw;
 663     } else {
 664       insn = &Assembler::str;
 665     }
 666     break;
 667   case T_CHAR:
 668   case T_SHORT:
 669     assert(c->as_jint() == 0, "should be");
 670     insn = &Assembler::strh;
 671     break;
 672   case T_BOOLEAN:
 673   case T_BYTE:
 674     assert(c->as_jint() == 0, "should be");
 675     insn = &Assembler::strb;
 676     break;
 677   default:
 678     ShouldNotReachHere();
 679     insn = &Assembler::str;  // unreachable
 680   }
 681 
 682   if (info) add_debug_info_for_null_check_here(info);
 683   (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
 684 }
 685 
 686 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 687   assert(src->is_register(), "should not call otherwise");
 688   assert(dest->is_register(), "should not call otherwise");
 689 
 690   // move between cpu-registers
 691   if (dest->is_single_cpu()) {
 692     if (src->type() == T_LONG) {
 693       // Can do LONG -> OBJECT
 694       move_regs(src->as_register_lo(), dest->as_register());
 695       return;
 696     }
 697     assert(src->is_single_cpu(), "must match");
 698     if (src->type() == T_OBJECT) {
 699       __ verify_oop(src->as_register());
 700     }
 701     move_regs(src->as_register(), dest->as_register());
 702 
 703   } else if (dest->is_double_cpu()) {
 704     if (is_reference_type(src->type())) {
 705       // Surprising to me but we can see move of a long to t_object
 706       __ verify_oop(src->as_register());
 707       move_regs(src->as_register(), dest->as_register_lo());
 708       return;
 709     }
 710     assert(src->is_double_cpu(), "must match");
 711     Register f_lo = src->as_register_lo();
 712     Register f_hi = src->as_register_hi();
 713     Register t_lo = dest->as_register_lo();
 714     Register t_hi = dest->as_register_hi();
 715     assert(f_hi == f_lo, "must be same");
 716     assert(t_hi == t_lo, "must be same");
 717     move_regs(f_lo, t_lo);
 718 
 719   } else if (dest->is_single_fpu()) {
 720     __ fmovs(dest->as_float_reg(), src->as_float_reg());
 721 
 722   } else if (dest->is_double_fpu()) {
 723     __ fmovd(dest->as_double_reg(), src->as_double_reg());
 724 
 725   } else {
 726     ShouldNotReachHere();
 727   }
 728 }
 729 
 730 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 731   precond(src->is_register() && dest->is_stack());
 732 
 733   uint const c_sz32 = sizeof(uint32_t);
 734   uint const c_sz64 = sizeof(uint64_t);
 735 
 736   if (src->is_single_cpu()) {
 737     int index = dest->single_stack_ix();
 738     if (is_reference_type(type)) {
 739       __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 740       __ verify_oop(src->as_register());
 741     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 742       __ str(src->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 743     } else {
 744       __ strw(src->as_register(), stack_slot_address(index, c_sz32, rscratch1));
 745     }
 746 
 747   } else if (src->is_double_cpu()) {
 748     int index = dest->double_stack_ix();
 749     Address dest_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
 750     __ str(src->as_register_lo(), dest_addr_LO);
 751 
 752   } else if (src->is_single_fpu()) {
 753     int index = dest->single_stack_ix();
 754     __ strs(src->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
 755 
 756   } else if (src->is_double_fpu()) {
 757     int index = dest->double_stack_ix();
 758     __ strd(src->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
 759 
 760   } else {
 761     ShouldNotReachHere();
 762   }
 763 }
 764 
 765 
 766 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) {
 767   LIR_Address* to_addr = dest->as_address_ptr();
 768   PatchingStub* patch = nullptr;
 769   Register compressed_src = rscratch1;
 770 
 771   if (patch_code != lir_patch_none) {
 772     deoptimize_trap(info);
 773     return;
 774   }
 775 
 776   if (is_reference_type(type)) {
 777     __ verify_oop(src->as_register());
 778 
 779     if (UseCompressedOops && !wide) {
 780       __ encode_heap_oop(compressed_src, src->as_register());
 781     } else {
 782       compressed_src = src->as_register();
 783     }
 784   }
 785 
 786   int null_check_here = code_offset();
 787   switch (type) {
 788     case T_FLOAT: {
 789       __ strs(src->as_float_reg(), as_Address(to_addr));
 790       break;
 791     }
 792 
 793     case T_DOUBLE: {
 794       __ strd(src->as_double_reg(), as_Address(to_addr));
 795       break;
 796     }
 797 
 798     case T_ARRAY:   // fall through
 799     case T_OBJECT:  // fall through
 800       if (UseCompressedOops && !wide) {
 801         __ strw(compressed_src, as_Address(to_addr, rscratch2));
 802       } else {
 803          __ str(compressed_src, as_Address(to_addr));
 804       }
 805       break;
 806     case T_METADATA:
 807       // We get here to store a method pointer to the stack to pass to
 808       // a dtrace runtime call. This can't work on 64 bit with
 809       // compressed klass ptrs: T_METADATA can be a compressed klass
 810       // ptr or a 64 bit method pointer.
 811       ShouldNotReachHere();
 812       __ str(src->as_register(), as_Address(to_addr));
 813       break;
 814     case T_ADDRESS:
 815       __ str(src->as_register(), as_Address(to_addr));
 816       break;
 817     case T_INT:
 818       __ strw(src->as_register(), as_Address(to_addr));
 819       break;
 820 
 821     case T_LONG: {
 822       __ str(src->as_register_lo(), as_Address_lo(to_addr));
 823       break;
 824     }
 825 
 826     case T_BYTE:    // fall through
 827     case T_BOOLEAN: {
 828       __ strb(src->as_register(), as_Address(to_addr));
 829       break;
 830     }
 831 
 832     case T_CHAR:    // fall through
 833     case T_SHORT:
 834       __ strh(src->as_register(), as_Address(to_addr));
 835       break;
 836 
 837     default:
 838       ShouldNotReachHere();
 839   }
 840   if (info != nullptr) {
 841     add_debug_info_for_null_check(null_check_here, info);
 842   }
 843 }
 844 
 845 
 846 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 847   precond(src->is_stack() && dest->is_register());
 848 
 849   uint const c_sz32 = sizeof(uint32_t);
 850   uint const c_sz64 = sizeof(uint64_t);
 851 
 852   if (dest->is_single_cpu()) {
 853     int index = src->single_stack_ix();
 854     if (is_reference_type(type)) {
 855       __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 856       __ verify_oop(dest->as_register());
 857     } else if (type == T_METADATA || type == T_ADDRESS) {
 858       __ ldr(dest->as_register(), stack_slot_address(index, c_sz64, rscratch1));
 859     } else {
 860       __ ldrw(dest->as_register(), stack_slot_address(index, c_sz32, rscratch1));
 861     }
 862 
 863   } else if (dest->is_double_cpu()) {
 864     int index = src->double_stack_ix();
 865     Address src_addr_LO = stack_slot_address(index, c_sz64, rscratch1, lo_word_offset_in_bytes);
 866     __ ldr(dest->as_register_lo(), src_addr_LO);
 867 
 868   } else if (dest->is_single_fpu()) {
 869     int index = src->single_stack_ix();
 870     __ ldrs(dest->as_float_reg(), stack_slot_address(index, c_sz32, rscratch1));
 871 
 872   } else if (dest->is_double_fpu()) {
 873     int index = src->double_stack_ix();
 874     __ ldrd(dest->as_double_reg(), stack_slot_address(index, c_sz64, rscratch1));
 875 
 876   } else {
 877     ShouldNotReachHere();
 878   }
 879 }
 880 
 881 
 882 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 883   address target = nullptr;
 884   relocInfo::relocType reloc_type = relocInfo::none;
 885 
 886   switch (patching_id(info)) {
 887   case PatchingStub::access_field_id:
 888     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 889     reloc_type = relocInfo::section_word_type;
 890     break;
 891   case PatchingStub::load_klass_id:
 892     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 893     reloc_type = relocInfo::metadata_type;
 894     break;
 895   case PatchingStub::load_mirror_id:
 896     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 897     reloc_type = relocInfo::oop_type;
 898     break;
 899   case PatchingStub::load_appendix_id:
 900     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 901     reloc_type = relocInfo::oop_type;
 902     break;
 903   default: ShouldNotReachHere();
 904   }
 905 
 906   __ far_call(RuntimeAddress(target));
 907   add_call_info_here(info);
 908 }
 909 
 910 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 911 
 912   LIR_Opr temp;
 913   if (type == T_LONG || type == T_DOUBLE)
 914     temp = FrameMap::rscratch1_long_opr;
 915   else
 916     temp = FrameMap::rscratch1_opr;
 917 
 918   stack2reg(src, temp, src->type());
 919   reg2stack(temp, dest, dest->type(), false);
 920 }
 921 
 922 
 923 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) {
 924   LIR_Address* addr = src->as_address_ptr();
 925   LIR_Address* from_addr = src->as_address_ptr();
 926 
 927   if (addr->base()->type() == T_OBJECT) {
 928     __ verify_oop(addr->base()->as_pointer_register());
 929   }
 930 
 931   if (patch_code != lir_patch_none) {
 932     deoptimize_trap(info);
 933     return;
 934   }
 935 
 936   if (info != nullptr) {
 937     add_debug_info_for_null_check_here(info);
 938   }
 939   int null_check_here = code_offset();
 940   switch (type) {
 941     case T_FLOAT: {
 942       __ ldrs(dest->as_float_reg(), as_Address(from_addr));
 943       break;
 944     }
 945 
 946     case T_DOUBLE: {
 947       __ ldrd(dest->as_double_reg(), as_Address(from_addr));
 948       break;
 949     }
 950 
 951     case T_ARRAY:   // fall through
 952     case T_OBJECT:  // fall through
 953       if (UseCompressedOops && !wide) {
 954         __ ldrw(dest->as_register(), as_Address(from_addr));
 955       } else {
 956         __ ldr(dest->as_register(), as_Address(from_addr));
 957       }
 958       break;
 959     case T_METADATA:
 960       // We get here to store a method pointer to the stack to pass to
 961       // a dtrace runtime call. This can't work on 64 bit with
 962       // compressed klass ptrs: T_METADATA can be a compressed klass
 963       // ptr or a 64 bit method pointer.
 964       ShouldNotReachHere();
 965       __ ldr(dest->as_register(), as_Address(from_addr));
 966       break;
 967     case T_ADDRESS:
 968       __ ldr(dest->as_register(), as_Address(from_addr));
 969       break;
 970     case T_INT:
 971       __ ldrw(dest->as_register(), as_Address(from_addr));
 972       break;
 973 
 974     case T_LONG: {
 975       __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
 976       break;
 977     }
 978 
 979     case T_BYTE:
 980       __ ldrsb(dest->as_register(), as_Address(from_addr));
 981       break;
 982     case T_BOOLEAN: {
 983       __ ldrb(dest->as_register(), as_Address(from_addr));
 984       break;
 985     }
 986 
 987     case T_CHAR:
 988       __ ldrh(dest->as_register(), as_Address(from_addr));
 989       break;
 990     case T_SHORT:
 991       __ ldrsh(dest->as_register(), as_Address(from_addr));
 992       break;
 993 
 994     default:
 995       ShouldNotReachHere();
 996   }
 997 
 998   if (is_reference_type(type)) {
 999     if (UseCompressedOops && !wide) {
1000       __ decode_heap_oop(dest->as_register());
1001     }
1002 
1003     if (!(UseZGC && !ZGenerational)) {
1004       // Load barrier has not yet been applied, so ZGC can't verify the oop here
1005       __ verify_oop(dest->as_register());
1006     }
1007   }
1008 }
1009 
1010 
1011 int LIR_Assembler::array_element_size(BasicType type) const {
1012   int elem_size = type2aelembytes(type);
1013   return exact_log2(elem_size);
1014 }
1015 
1016 
1017 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1018   switch (op->code()) {
1019   case lir_idiv:
1020   case lir_irem:
1021     arithmetic_idiv(op->code(),
1022                     op->in_opr1(),
1023                     op->in_opr2(),
1024                     op->in_opr3(),
1025                     op->result_opr(),
1026                     op->info());
1027     break;
1028   case lir_fmad:
1029     __ fmaddd(op->result_opr()->as_double_reg(),
1030               op->in_opr1()->as_double_reg(),
1031               op->in_opr2()->as_double_reg(),
1032               op->in_opr3()->as_double_reg());
1033     break;
1034   case lir_fmaf:
1035     __ fmadds(op->result_opr()->as_float_reg(),
1036               op->in_opr1()->as_float_reg(),
1037               op->in_opr2()->as_float_reg(),
1038               op->in_opr3()->as_float_reg());
1039     break;
1040   default:      ShouldNotReachHere(); break;
1041   }
1042 }
1043 
1044 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1045 #ifdef ASSERT
1046   assert(op->block() == nullptr || op->block()->label() == op->label(), "wrong label");
1047   if (op->block() != nullptr)  _branch_target_blocks.append(op->block());
1048   if (op->ublock() != nullptr) _branch_target_blocks.append(op->ublock());
1049 #endif
1050 
1051   if (op->cond() == lir_cond_always) {
1052     if (op->info() != nullptr) add_debug_info_for_branch(op->info());
1053     __ b(*(op->label()));
1054   } else {
1055     Assembler::Condition acond;
1056     if (op->code() == lir_cond_float_branch) {
1057       bool is_unordered = (op->ublock() == op->block());
1058       // Assembler::EQ does not permit unordered branches, so we add
1059       // another branch here.  Likewise, Assembler::NE does not permit
1060       // ordered branches.
1061       if ((is_unordered && op->cond() == lir_cond_equal)
1062           || (!is_unordered && op->cond() == lir_cond_notEqual))
1063         __ br(Assembler::VS, *(op->ublock()->label()));
1064       switch(op->cond()) {
1065       case lir_cond_equal:        acond = Assembler::EQ; break;
1066       case lir_cond_notEqual:     acond = Assembler::NE; break;
1067       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1068       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1069       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1070       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1071       default:                    ShouldNotReachHere();
1072         acond = Assembler::EQ;  // unreachable
1073       }
1074     } else {
1075       switch (op->cond()) {
1076         case lir_cond_equal:        acond = Assembler::EQ; break;
1077         case lir_cond_notEqual:     acond = Assembler::NE; break;
1078         case lir_cond_less:         acond = Assembler::LT; break;
1079         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1080         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1081         case lir_cond_greater:      acond = Assembler::GT; break;
1082         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1083         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1084         default:                    ShouldNotReachHere();
1085           acond = Assembler::EQ;  // unreachable
1086       }
1087     }
1088     __ br(acond,*(op->label()));
1089   }
1090 }
1091 
1092 
1093 
1094 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1095   LIR_Opr src  = op->in_opr();
1096   LIR_Opr dest = op->result_opr();
1097 
1098   switch (op->bytecode()) {
1099     case Bytecodes::_i2f:
1100       {
1101         __ scvtfws(dest->as_float_reg(), src->as_register());
1102         break;
1103       }
1104     case Bytecodes::_i2d:
1105       {
1106         __ scvtfwd(dest->as_double_reg(), src->as_register());
1107         break;
1108       }
1109     case Bytecodes::_l2d:
1110       {
1111         __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1112         break;
1113       }
1114     case Bytecodes::_l2f:
1115       {
1116         __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1117         break;
1118       }
1119     case Bytecodes::_f2d:
1120       {
1121         __ fcvts(dest->as_double_reg(), src->as_float_reg());
1122         break;
1123       }
1124     case Bytecodes::_d2f:
1125       {
1126         __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1127         break;
1128       }
1129     case Bytecodes::_i2c:
1130       {
1131         __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1132         break;
1133       }
1134     case Bytecodes::_i2l:
1135       {
1136         __ sxtw(dest->as_register_lo(), src->as_register());
1137         break;
1138       }
1139     case Bytecodes::_i2s:
1140       {
1141         __ sxth(dest->as_register(), src->as_register());
1142         break;
1143       }
1144     case Bytecodes::_i2b:
1145       {
1146         __ sxtb(dest->as_register(), src->as_register());
1147         break;
1148       }
1149     case Bytecodes::_l2i:
1150       {
1151         _masm->block_comment("FIXME: This could be a no-op");
1152         __ uxtw(dest->as_register(), src->as_register_lo());
1153         break;
1154       }
1155     case Bytecodes::_d2l:
1156       {
1157         __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1158         break;
1159       }
1160     case Bytecodes::_f2i:
1161       {
1162         __ fcvtzsw(dest->as_register(), src->as_float_reg());
1163         break;
1164       }
1165     case Bytecodes::_f2l:
1166       {
1167         __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1168         break;
1169       }
1170     case Bytecodes::_d2i:
1171       {
1172         __ fcvtzdw(dest->as_register(), src->as_double_reg());
1173         break;
1174       }
1175     default: ShouldNotReachHere();
1176   }
1177 }
1178 
1179 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1180   if (op->init_check()) {
1181     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1182                                InstanceKlass::init_state_offset()));
1183     __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1184     add_debug_info_for_null_check_here(op->stub()->info());
1185     __ br(Assembler::NE, *op->stub()->entry());
1186   }
1187   __ allocate_object(op->obj()->as_register(),
1188                      op->tmp1()->as_register(),
1189                      op->tmp2()->as_register(),
1190                      op->header_size(),
1191                      op->object_size(),
1192                      op->klass()->as_register(),
1193                      *op->stub()->entry());
1194   __ bind(*op->stub()->continuation());
1195 }
1196 
1197 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1198   Register len =  op->len()->as_register();
1199   __ uxtw(len, len);
1200 
1201   if (UseSlowPath ||
1202       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1203       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1204     __ b(*op->stub()->entry());
1205   } else {
1206     Register tmp1 = op->tmp1()->as_register();
1207     Register tmp2 = op->tmp2()->as_register();
1208     Register tmp3 = op->tmp3()->as_register();
1209     if (len == tmp1) {
1210       tmp1 = tmp3;
1211     } else if (len == tmp2) {
1212       tmp2 = tmp3;
1213     } else if (len == tmp3) {
1214       // everything is ok
1215     } else {
1216       __ mov(tmp3, len);
1217     }
1218     __ allocate_array(op->obj()->as_register(),
1219                       len,
1220                       tmp1,
1221                       tmp2,
1222                       arrayOopDesc::header_size(op->type()),
1223                       array_element_size(op->type()),
1224                       op->klass()->as_register(),
1225                       *op->stub()->entry());
1226   }
1227   __ bind(*op->stub()->continuation());
1228 }
1229 
1230 void LIR_Assembler::type_profile_helper(Register mdo,
1231                                         ciMethodData *md, ciProfileData *data,
1232                                         Register recv, Label* update_done) {
1233   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1234     Label next_test;
1235     // See if the receiver is receiver[n].
1236     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1237     __ ldr(rscratch1, Address(rscratch2));
1238     __ cmp(recv, rscratch1);
1239     __ br(Assembler::NE, next_test);
1240     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1241     __ addptr(data_addr, DataLayout::counter_increment);
1242     __ b(*update_done);
1243     __ bind(next_test);
1244   }
1245 
1246   // Didn't find receiver; find next empty slot and fill it in
1247   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1248     Label next_test;
1249     __ lea(rscratch2,
1250            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1251     Address recv_addr(rscratch2);
1252     __ ldr(rscratch1, recv_addr);
1253     __ cbnz(rscratch1, next_test);
1254     __ str(recv, recv_addr);
1255     __ mov(rscratch1, DataLayout::counter_increment);
1256     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1257     __ str(rscratch1, Address(rscratch2));
1258     __ b(*update_done);
1259     __ bind(next_test);
1260   }
1261 }
1262 
1263 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1264   // we always need a stub for the failure case.
1265   CodeStub* stub = op->stub();
1266   Register obj = op->object()->as_register();
1267   Register k_RInfo = op->tmp1()->as_register();
1268   Register klass_RInfo = op->tmp2()->as_register();
1269   Register dst = op->result_opr()->as_register();
1270   ciKlass* k = op->klass();
1271   Register Rtmp1 = noreg;
1272 
1273   // check if it needs to be profiled
1274   ciMethodData* md;
1275   ciProfileData* data;
1276 
1277   const bool should_profile = op->should_profile();
1278 
1279   if (should_profile) {
1280     ciMethod* method = op->profiled_method();
1281     assert(method != nullptr, "Should have method");
1282     int bci = op->profiled_bci();
1283     md = method->method_data_or_null();
1284     assert(md != nullptr, "Sanity");
1285     data = md->bci_to_data(bci);
1286     assert(data != nullptr,                "need data for type check");
1287     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1288   }
1289   Label* success_target = success;
1290   Label* failure_target = failure;
1291 
1292   if (obj == k_RInfo) {
1293     k_RInfo = dst;
1294   } else if (obj == klass_RInfo) {
1295     klass_RInfo = dst;
1296   }
1297   if (k->is_loaded() && !UseCompressedClassPointers) {
1298     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1299   } else {
1300     Rtmp1 = op->tmp3()->as_register();
1301     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1302   }
1303 
1304   assert_different_registers(obj, k_RInfo, klass_RInfo);
1305 
1306   if (should_profile) {
1307     Register mdo  = klass_RInfo;
1308     __ mov_metadata(mdo, md->constant_encoding());
1309     Label not_null;
1310     __ cbnz(obj, not_null);
1311     // Object is null; update MDO and exit
1312     Address data_addr
1313       = __ form_address(rscratch2, mdo,
1314                         md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1315                         0);
1316     __ ldrb(rscratch1, data_addr);
1317     __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1318     __ strb(rscratch1, data_addr);
1319     __ b(*obj_is_null);
1320     __ bind(not_null);
1321 
1322     Label update_done;
1323     Register recv = k_RInfo;
1324     __ load_klass(recv, obj);
1325     type_profile_helper(mdo, md, data, recv, &update_done);
1326     Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1327     __ addptr(counter_addr, DataLayout::counter_increment);
1328 
1329     __ bind(update_done);
1330   } else {
1331     __ cbz(obj, *obj_is_null);
1332   }
1333 
1334   if (!k->is_loaded()) {
1335     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1336   } else {
1337     __ mov_metadata(k_RInfo, k->constant_encoding());
1338   }
1339   __ verify_oop(obj);
1340 
1341   if (op->fast_check()) {
1342     // get object class
1343     // not a safepoint as obj null check happens earlier
1344     __ load_klass(rscratch1, obj);
1345     __ cmp( rscratch1, k_RInfo);
1346 
1347     __ br(Assembler::NE, *failure_target);
1348     // successful cast, fall through to profile or jump
1349   } else {
1350     // get object class
1351     // not a safepoint as obj null check happens earlier
1352     __ load_klass(klass_RInfo, obj);
1353     if (k->is_loaded()) {
1354       // See if we get an immediate positive hit
1355       __ ldr(rscratch1, Address(klass_RInfo, int64_t(k->super_check_offset())));
1356       __ cmp(k_RInfo, rscratch1);
1357       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1358         __ br(Assembler::NE, *failure_target);
1359         // successful cast, fall through to profile or jump
1360       } else {
1361         // See if we get an immediate positive hit
1362         __ br(Assembler::EQ, *success_target);
1363         // check for self
1364         __ cmp(klass_RInfo, k_RInfo);
1365         __ br(Assembler::EQ, *success_target);
1366 
1367         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1368         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1369         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1370         // result is a boolean
1371         __ cbzw(klass_RInfo, *failure_target);
1372         // successful cast, fall through to profile or jump
1373       }
1374     } else {
1375       // perform the fast part of the checking logic
1376       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1377       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1378       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1379       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1380       __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1381       // result is a boolean
1382       __ cbz(k_RInfo, *failure_target);
1383       // successful cast, fall through to profile or jump
1384     }
1385   }
1386   __ b(*success);
1387 }
1388 
1389 
1390 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1391   const bool should_profile = op->should_profile();
1392 
1393   LIR_Code code = op->code();
1394   if (code == lir_store_check) {
1395     Register value = op->object()->as_register();
1396     Register array = op->array()->as_register();
1397     Register k_RInfo = op->tmp1()->as_register();
1398     Register klass_RInfo = op->tmp2()->as_register();
1399     Register Rtmp1 = op->tmp3()->as_register();
1400 
1401     CodeStub* stub = op->stub();
1402 
1403     // check if it needs to be profiled
1404     ciMethodData* md;
1405     ciProfileData* data;
1406 
1407     if (should_profile) {
1408       ciMethod* method = op->profiled_method();
1409       assert(method != nullptr, "Should have method");
1410       int bci = op->profiled_bci();
1411       md = method->method_data_or_null();
1412       assert(md != nullptr, "Sanity");
1413       data = md->bci_to_data(bci);
1414       assert(data != nullptr,                "need data for type check");
1415       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1416     }
1417     Label done;
1418     Label* success_target = &done;
1419     Label* failure_target = stub->entry();
1420 
1421     if (should_profile) {
1422       Label not_null;
1423       Register mdo  = klass_RInfo;
1424       __ mov_metadata(mdo, md->constant_encoding());
1425       __ cbnz(value, not_null);
1426       // Object is null; update MDO and exit
1427       Address data_addr
1428         = __ form_address(rscratch2, mdo,
1429                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1430                           0);
1431       __ ldrb(rscratch1, data_addr);
1432       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1433       __ strb(rscratch1, data_addr);
1434       __ b(done);
1435       __ bind(not_null);
1436 
1437       Label update_done;
1438       Register recv = k_RInfo;
1439       __ load_klass(recv, value);
1440       type_profile_helper(mdo, md, data, recv, &update_done);
1441       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1442       __ addptr(counter_addr, DataLayout::counter_increment);
1443       __ bind(update_done);
1444     } else {
1445       __ cbz(value, done);
1446     }
1447 
1448     add_debug_info_for_null_check_here(op->info_for_exception());
1449     __ load_klass(k_RInfo, array);
1450     __ load_klass(klass_RInfo, value);
1451 
1452     // get instance klass (it's already uncompressed)
1453     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1454     // perform the fast part of the checking logic
1455     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr);
1456     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1457     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1458     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1459     __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1460     // result is a boolean
1461     __ cbzw(k_RInfo, *failure_target);
1462     // fall through to the success case
1463 
1464     __ bind(done);
1465   } else if (code == lir_checkcast) {
1466     Register obj = op->object()->as_register();
1467     Register dst = op->result_opr()->as_register();
1468     Label success;
1469     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1470     __ bind(success);
1471     if (dst != obj) {
1472       __ mov(dst, obj);
1473     }
1474   } else if (code == lir_instanceof) {
1475     Register obj = op->object()->as_register();
1476     Register dst = op->result_opr()->as_register();
1477     Label success, failure, done;
1478     emit_typecheck_helper(op, &success, &failure, &failure);
1479     __ bind(failure);
1480     __ mov(dst, zr);
1481     __ b(done);
1482     __ bind(success);
1483     __ mov(dst, 1);
1484     __ bind(done);
1485   } else {
1486     ShouldNotReachHere();
1487   }
1488 }
1489 
1490 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1491   __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1492   __ cset(rscratch1, Assembler::NE);
1493   __ membar(__ AnyAny);
1494 }
1495 
1496 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1497   __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1498   __ cset(rscratch1, Assembler::NE);
1499   __ membar(__ AnyAny);
1500 }
1501 
1502 
1503 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1504   Register addr;
1505   if (op->addr()->is_register()) {
1506     addr = as_reg(op->addr());
1507   } else {
1508     assert(op->addr()->is_address(), "what else?");
1509     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1510     assert(addr_ptr->disp() == 0, "need 0 disp");
1511     assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index");
1512     addr = as_reg(addr_ptr->base());
1513   }
1514   Register newval = as_reg(op->new_value());
1515   Register cmpval = as_reg(op->cmp_value());
1516 
1517   if (op->code() == lir_cas_obj) {
1518     if (UseCompressedOops) {
1519       Register t1 = op->tmp1()->as_register();
1520       assert(op->tmp1()->is_valid(), "must be");
1521       __ encode_heap_oop(t1, cmpval);
1522       cmpval = t1;
1523       __ encode_heap_oop(rscratch2, newval);
1524       newval = rscratch2;
1525       casw(addr, newval, cmpval);
1526     } else {
1527       casl(addr, newval, cmpval);
1528     }
1529   } else if (op->code() == lir_cas_int) {
1530     casw(addr, newval, cmpval);
1531   } else {
1532     casl(addr, newval, cmpval);
1533   }
1534 }
1535 
1536 
1537 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1538                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1539   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on aarch64");
1540 
1541   Assembler::Condition acond, ncond;
1542   switch (condition) {
1543   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1544   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1545   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1546   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1547   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1548   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1549   case lir_cond_belowEqual:
1550   case lir_cond_aboveEqual:
1551   default:                    ShouldNotReachHere();
1552     acond = Assembler::EQ; ncond = Assembler::NE;  // unreachable
1553   }
1554 
1555   assert(result->is_single_cpu() || result->is_double_cpu(),
1556          "expect single register for result");
1557   if (opr1->is_constant() && opr2->is_constant()
1558       && opr1->type() == T_INT && opr2->type() == T_INT) {
1559     jint val1 = opr1->as_jint();
1560     jint val2 = opr2->as_jint();
1561     if (val1 == 0 && val2 == 1) {
1562       __ cset(result->as_register(), ncond);
1563       return;
1564     } else if (val1 == 1 && val2 == 0) {
1565       __ cset(result->as_register(), acond);
1566       return;
1567     }
1568   }
1569 
1570   if (opr1->is_constant() && opr2->is_constant()
1571       && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1572     jlong val1 = opr1->as_jlong();
1573     jlong val2 = opr2->as_jlong();
1574     if (val1 == 0 && val2 == 1) {
1575       __ cset(result->as_register_lo(), ncond);
1576       return;
1577     } else if (val1 == 1 && val2 == 0) {
1578       __ cset(result->as_register_lo(), acond);
1579       return;
1580     }
1581   }
1582 
1583   if (opr1->is_stack()) {
1584     stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1585     opr1 = FrameMap::rscratch1_opr;
1586   } else if (opr1->is_constant()) {
1587     LIR_Opr tmp
1588       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1589     const2reg(opr1, tmp, lir_patch_none, nullptr);
1590     opr1 = tmp;
1591   }
1592 
1593   if (opr2->is_stack()) {
1594     stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1595     opr2 = FrameMap::rscratch2_opr;
1596   } else if (opr2->is_constant()) {
1597     LIR_Opr tmp
1598       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1599     const2reg(opr2, tmp, lir_patch_none, nullptr);
1600     opr2 = tmp;
1601   }
1602 
1603   if (result->type() == T_LONG)
1604     __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1605   else
1606     __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1607 }
1608 
1609 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1610   assert(info == nullptr, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1611 
1612   if (left->is_single_cpu()) {
1613     Register lreg = left->as_register();
1614     Register dreg = as_reg(dest);
1615 
1616     if (right->is_single_cpu()) {
1617       // cpu register - cpu register
1618 
1619       assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1620              "should be");
1621       Register rreg = right->as_register();
1622       switch (code) {
1623       case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1624       case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1625       case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1626       default:      ShouldNotReachHere();
1627       }
1628 
1629     } else if (right->is_double_cpu()) {
1630       Register rreg = right->as_register_lo();
1631       // single_cpu + double_cpu: can happen with obj+long
1632       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1633       switch (code) {
1634       case lir_add: __ add(dreg, lreg, rreg); break;
1635       case lir_sub: __ sub(dreg, lreg, rreg); break;
1636       default: ShouldNotReachHere();
1637       }
1638     } else if (right->is_constant()) {
1639       // cpu register - constant
1640       jlong c;
1641 
1642       // FIXME.  This is fugly: we really need to factor all this logic.
1643       switch(right->type()) {
1644       case T_LONG:
1645         c = right->as_constant_ptr()->as_jlong();
1646         break;
1647       case T_INT:
1648       case T_ADDRESS:
1649         c = right->as_constant_ptr()->as_jint();
1650         break;
1651       default:
1652         ShouldNotReachHere();
1653         c = 0;  // unreachable
1654         break;
1655       }
1656 
1657       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1658       if (c == 0 && dreg == lreg) {
1659         COMMENT("effective nop elided");
1660         return;
1661       }
1662       switch(left->type()) {
1663       case T_INT:
1664         switch (code) {
1665         case lir_add: __ addw(dreg, lreg, c); break;
1666         case lir_sub: __ subw(dreg, lreg, c); break;
1667         default: ShouldNotReachHere();
1668         }
1669         break;
1670       case T_OBJECT:
1671       case T_ADDRESS:
1672         switch (code) {
1673         case lir_add: __ add(dreg, lreg, c); break;
1674         case lir_sub: __ sub(dreg, lreg, c); break;
1675         default: ShouldNotReachHere();
1676         }
1677         break;
1678       default:
1679         ShouldNotReachHere();
1680       }
1681     } else {
1682       ShouldNotReachHere();
1683     }
1684 
1685   } else if (left->is_double_cpu()) {
1686     Register lreg_lo = left->as_register_lo();
1687 
1688     if (right->is_double_cpu()) {
1689       // cpu register - cpu register
1690       Register rreg_lo = right->as_register_lo();
1691       switch (code) {
1692       case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1693       case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1694       case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1695       case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1696       case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1697       default:
1698         ShouldNotReachHere();
1699       }
1700 
1701     } else if (right->is_constant()) {
1702       jlong c = right->as_constant_ptr()->as_jlong();
1703       Register dreg = as_reg(dest);
1704       switch (code) {
1705         case lir_add:
1706         case lir_sub:
1707           if (c == 0 && dreg == lreg_lo) {
1708             COMMENT("effective nop elided");
1709             return;
1710           }
1711           code == lir_add ? __ add(dreg, lreg_lo, c) : __ sub(dreg, lreg_lo, c);
1712           break;
1713         case lir_div:
1714           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1715           if (c == 1) {
1716             // move lreg_lo to dreg if divisor is 1
1717             __ mov(dreg, lreg_lo);
1718           } else {
1719             unsigned int shift = log2i_exact(c);
1720             // use rscratch1 as intermediate result register
1721             __ asr(rscratch1, lreg_lo, 63);
1722             __ add(rscratch1, lreg_lo, rscratch1, Assembler::LSR, 64 - shift);
1723             __ asr(dreg, rscratch1, shift);
1724           }
1725           break;
1726         case lir_rem:
1727           assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1728           if (c == 1) {
1729             // move 0 to dreg if divisor is 1
1730             __ mov(dreg, zr);
1731           } else {
1732             // use rscratch1 as intermediate result register
1733             __ negs(rscratch1, lreg_lo);
1734             __ andr(dreg, lreg_lo, c - 1);
1735             __ andr(rscratch1, rscratch1, c - 1);
1736             __ csneg(dreg, dreg, rscratch1, Assembler::MI);
1737           }
1738           break;
1739         default:
1740           ShouldNotReachHere();
1741       }
1742     } else {
1743       ShouldNotReachHere();
1744     }
1745   } else if (left->is_single_fpu()) {
1746     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1747     switch (code) {
1748     case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1749     case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1750     case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1751     case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1752     default:
1753       ShouldNotReachHere();
1754     }
1755   } else if (left->is_double_fpu()) {
1756     if (right->is_double_fpu()) {
1757       // fpu register - fpu register
1758       switch (code) {
1759       case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1760       case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1761       case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1762       case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1763       default:
1764         ShouldNotReachHere();
1765       }
1766     } else {
1767       if (right->is_constant()) {
1768         ShouldNotReachHere();
1769       }
1770       ShouldNotReachHere();
1771     }
1772   } else if (left->is_single_stack() || left->is_address()) {
1773     assert(left == dest, "left and dest must be equal");
1774     ShouldNotReachHere();
1775   } else {
1776     ShouldNotReachHere();
1777   }
1778 }
1779 
1780 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1781 
1782 
1783 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr tmp, LIR_Opr dest, LIR_Op* op) {
1784   switch(code) {
1785   case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1786   case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1787   case lir_f2hf: __ flt_to_flt16(dest->as_register(), value->as_float_reg(), tmp->as_float_reg()); break;
1788   case lir_hf2f: __ flt16_to_flt(dest->as_float_reg(), value->as_register(), tmp->as_float_reg()); break;
1789   default      : ShouldNotReachHere();
1790   }
1791 }
1792 
1793 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1794 
1795   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1796   Register Rleft = left->is_single_cpu() ? left->as_register() :
1797                                            left->as_register_lo();
1798    if (dst->is_single_cpu()) {
1799      Register Rdst = dst->as_register();
1800      if (right->is_constant()) {
1801        switch (code) {
1802          case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1803          case lir_logic_or:  __ orrw (Rdst, Rleft, right->as_jint()); break;
1804          case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1805          default: ShouldNotReachHere(); break;
1806        }
1807      } else {
1808        Register Rright = right->is_single_cpu() ? right->as_register() :
1809                                                   right->as_register_lo();
1810        switch (code) {
1811          case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1812          case lir_logic_or:  __ orrw (Rdst, Rleft, Rright); break;
1813          case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1814          default: ShouldNotReachHere(); break;
1815        }
1816      }
1817    } else {
1818      Register Rdst = dst->as_register_lo();
1819      if (right->is_constant()) {
1820        switch (code) {
1821          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1822          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jlong()); break;
1823          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1824          default: ShouldNotReachHere(); break;
1825        }
1826      } else {
1827        Register Rright = right->is_single_cpu() ? right->as_register() :
1828                                                   right->as_register_lo();
1829        switch (code) {
1830          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1831          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
1832          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1833          default: ShouldNotReachHere(); break;
1834        }
1835      }
1836    }
1837 }
1838 
1839 
1840 
1841 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr illegal, LIR_Opr result, CodeEmitInfo* info) {
1842 
1843   // opcode check
1844   assert((code == lir_idiv) || (code == lir_irem), "opcode must be idiv or irem");
1845   bool is_irem = (code == lir_irem);
1846 
1847   // operand check
1848   assert(left->is_single_cpu(),   "left must be register");
1849   assert(right->is_single_cpu() || right->is_constant(),  "right must be register or constant");
1850   assert(result->is_single_cpu(), "result must be register");
1851   Register lreg = left->as_register();
1852   Register dreg = result->as_register();
1853 
1854   // power-of-2 constant check and codegen
1855   if (right->is_constant()) {
1856     int c = right->as_constant_ptr()->as_jint();
1857     assert(c > 0 && is_power_of_2(c), "divisor must be power-of-2 constant");
1858     if (is_irem) {
1859       if (c == 1) {
1860         // move 0 to dreg if divisor is 1
1861         __ movw(dreg, zr);
1862       } else {
1863         // use rscratch1 as intermediate result register
1864         __ negsw(rscratch1, lreg);
1865         __ andw(dreg, lreg, c - 1);
1866         __ andw(rscratch1, rscratch1, c - 1);
1867         __ csnegw(dreg, dreg, rscratch1, Assembler::MI);
1868       }
1869     } else {
1870       if (c == 1) {
1871         // move lreg to dreg if divisor is 1
1872         __ movw(dreg, lreg);
1873       } else {
1874         unsigned int shift = exact_log2(c);
1875         // use rscratch1 as intermediate result register
1876         __ asrw(rscratch1, lreg, 31);
1877         __ addw(rscratch1, lreg, rscratch1, Assembler::LSR, 32 - shift);
1878         __ asrw(dreg, rscratch1, shift);
1879       }
1880     }
1881   } else {
1882     Register rreg = right->as_register();
1883     __ corrected_idivl(dreg, lreg, rreg, is_irem, rscratch1);
1884   }
1885 }
1886 
1887 
1888 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1889   if (opr1->is_constant() && opr2->is_single_cpu()) {
1890     // tableswitch
1891     Register reg = as_reg(opr2);
1892     struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1893     __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1894   } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1895     Register reg1 = as_reg(opr1);
1896     if (opr2->is_single_cpu()) {
1897       // cpu register - cpu register
1898       Register reg2 = opr2->as_register();
1899       if (is_reference_type(opr1->type())) {
1900         __ cmpoop(reg1, reg2);
1901       } else {
1902         assert(!is_reference_type(opr2->type()), "cmp int, oop?");
1903         __ cmpw(reg1, reg2);
1904       }
1905       return;
1906     }
1907     if (opr2->is_double_cpu()) {
1908       // cpu register - cpu register
1909       Register reg2 = opr2->as_register_lo();
1910       __ cmp(reg1, reg2);
1911       return;
1912     }
1913 
1914     if (opr2->is_constant()) {
1915       bool is_32bit = false; // width of register operand
1916       jlong imm;
1917 
1918       switch(opr2->type()) {
1919       case T_INT:
1920         imm = opr2->as_constant_ptr()->as_jint();
1921         is_32bit = true;
1922         break;
1923       case T_LONG:
1924         imm = opr2->as_constant_ptr()->as_jlong();
1925         break;
1926       case T_ADDRESS:
1927         imm = opr2->as_constant_ptr()->as_jint();
1928         break;
1929       case T_METADATA:
1930         imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
1931         break;
1932       case T_OBJECT:
1933       case T_ARRAY:
1934         jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
1935         __ cmpoop(reg1, rscratch1);
1936         return;
1937       default:
1938         ShouldNotReachHere();
1939         imm = 0;  // unreachable
1940         break;
1941       }
1942 
1943       if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1944         if (is_32bit)
1945           __ cmpw(reg1, imm);
1946         else
1947           __ subs(zr, reg1, imm);
1948         return;
1949       } else {
1950         __ mov(rscratch1, imm);
1951         if (is_32bit)
1952           __ cmpw(reg1, rscratch1);
1953         else
1954           __ cmp(reg1, rscratch1);
1955         return;
1956       }
1957     } else
1958       ShouldNotReachHere();
1959   } else if (opr1->is_single_fpu()) {
1960     FloatRegister reg1 = opr1->as_float_reg();
1961     assert(opr2->is_single_fpu(), "expect single float register");
1962     FloatRegister reg2 = opr2->as_float_reg();
1963     __ fcmps(reg1, reg2);
1964   } else if (opr1->is_double_fpu()) {
1965     FloatRegister reg1 = opr1->as_double_reg();
1966     assert(opr2->is_double_fpu(), "expect double float register");
1967     FloatRegister reg2 = opr2->as_double_reg();
1968     __ fcmpd(reg1, reg2);
1969   } else {
1970     ShouldNotReachHere();
1971   }
1972 }
1973 
1974 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1975   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1976     bool is_unordered_less = (code == lir_ucmp_fd2i);
1977     if (left->is_single_fpu()) {
1978       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1979     } else if (left->is_double_fpu()) {
1980       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1981     } else {
1982       ShouldNotReachHere();
1983     }
1984   } else if (code == lir_cmp_l2i) {
1985     Label done;
1986     __ cmp(left->as_register_lo(), right->as_register_lo());
1987     __ mov(dst->as_register(), (uint64_t)-1L);
1988     __ br(Assembler::LT, done);
1989     __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1990     __ bind(done);
1991   } else {
1992     ShouldNotReachHere();
1993   }
1994 }
1995 
1996 
1997 void LIR_Assembler::align_call(LIR_Code code) {  }
1998 
1999 
2000 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2001   address call = __ trampoline_call(Address(op->addr(), rtype));
2002   if (call == nullptr) {
2003     bailout("trampoline stub overflow");
2004     return;
2005   }
2006   add_call_info(code_offset(), op->info());
2007   __ post_call_nop();
2008 }
2009 
2010 
2011 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2012   address call = __ ic_call(op->addr());
2013   if (call == nullptr) {
2014     bailout("trampoline stub overflow");
2015     return;
2016   }
2017   add_call_info(code_offset(), op->info());
2018   __ post_call_nop();
2019 }
2020 
2021 void LIR_Assembler::emit_static_call_stub() {
2022   address call_pc = __ pc();
2023   address stub = __ start_a_stub(call_stub_size());
2024   if (stub == nullptr) {
2025     bailout("static call stub overflow");
2026     return;
2027   }
2028 
2029   int start = __ offset();
2030 
2031   __ relocate(static_stub_Relocation::spec(call_pc));
2032   __ emit_static_call_stub();
2033 
2034   assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size()
2035         <= call_stub_size(), "stub too big");
2036   __ end_a_stub();
2037 }
2038 
2039 
2040 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2041   assert(exceptionOop->as_register() == r0, "must match");
2042   assert(exceptionPC->as_register() == r3, "must match");
2043 
2044   // exception object is not added to oop map by LinearScan
2045   // (LinearScan assumes that no oops are in fixed registers)
2046   info->add_register_oop(exceptionOop);
2047   Runtime1::StubID unwind_id;
2048 
2049   // get current pc information
2050   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2051   if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) {
2052     // As no instructions have been generated yet for this LIR node it's
2053     // possible that an oop map already exists for the current offset.
2054     // In that case insert an dummy NOP here to ensure all oop map PCs
2055     // are unique. See JDK-8237483.
2056     __ nop();
2057   }
2058   int pc_for_athrow_offset = __ offset();
2059   InternalAddress pc_for_athrow(__ pc());
2060   __ adr(exceptionPC->as_register(), pc_for_athrow);
2061   add_call_info(pc_for_athrow_offset, info); // for exception handler
2062 
2063   __ verify_not_null_oop(r0);
2064   // search an exception handler (r0: exception oop, r3: throwing pc)
2065   if (compilation()->has_fpu_code()) {
2066     unwind_id = Runtime1::handle_exception_id;
2067   } else {
2068     unwind_id = Runtime1::handle_exception_nofpu_id;
2069   }
2070   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2071 
2072   // FIXME: enough room for two byte trap   ????
2073   __ nop();
2074 }
2075 
2076 
2077 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2078   assert(exceptionOop->as_register() == r0, "must match");
2079 
2080   __ b(_unwind_handler_entry);
2081 }
2082 
2083 
2084 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2085   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2086   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2087 
2088   switch (left->type()) {
2089     case T_INT: {
2090       switch (code) {
2091       case lir_shl:  __ lslvw (dreg, lreg, count->as_register()); break;
2092       case lir_shr:  __ asrvw (dreg, lreg, count->as_register()); break;
2093       case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2094       default:
2095         ShouldNotReachHere();
2096         break;
2097       }
2098       break;
2099     case T_LONG:
2100     case T_ADDRESS:
2101     case T_OBJECT:
2102       switch (code) {
2103       case lir_shl:  __ lslv (dreg, lreg, count->as_register()); break;
2104       case lir_shr:  __ asrv (dreg, lreg, count->as_register()); break;
2105       case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2106       default:
2107         ShouldNotReachHere();
2108         break;
2109       }
2110       break;
2111     default:
2112       ShouldNotReachHere();
2113       break;
2114     }
2115   }
2116 }
2117 
2118 
2119 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2120   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2121   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2122 
2123   switch (left->type()) {
2124     case T_INT: {
2125       switch (code) {
2126       case lir_shl:  __ lslw (dreg, lreg, count); break;
2127       case lir_shr:  __ asrw (dreg, lreg, count); break;
2128       case lir_ushr: __ lsrw (dreg, lreg, count); break;
2129       default:
2130         ShouldNotReachHere();
2131         break;
2132       }
2133       break;
2134     case T_LONG:
2135     case T_ADDRESS:
2136     case T_OBJECT:
2137       switch (code) {
2138       case lir_shl:  __ lsl (dreg, lreg, count); break;
2139       case lir_shr:  __ asr (dreg, lreg, count); break;
2140       case lir_ushr: __ lsr (dreg, lreg, count); break;
2141       default:
2142         ShouldNotReachHere();
2143         break;
2144       }
2145       break;
2146     default:
2147       ShouldNotReachHere();
2148       break;
2149     }
2150   }
2151 }
2152 
2153 
2154 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2155   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2156   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2157   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2158   __ str (r, Address(sp, offset_from_rsp_in_bytes));
2159 }
2160 
2161 
2162 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2163   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2164   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2165   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2166   __ mov (rscratch1, c);
2167   __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2168 }
2169 
2170 
2171 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2172   ShouldNotReachHere();
2173   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2174   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2175   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2176   __ lea(rscratch1, __ constant_oop_address(o));
2177   __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2178 }
2179 
2180 
2181 // This code replaces a call to arraycopy; no exception may
2182 // be thrown in this code, they must be thrown in the System.arraycopy
2183 // activation frame; we could save some checks if this would not be the case
2184 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2185   ciArrayKlass* default_type = op->expected_type();
2186   Register src = op->src()->as_register();
2187   Register dst = op->dst()->as_register();
2188   Register src_pos = op->src_pos()->as_register();
2189   Register dst_pos = op->dst_pos()->as_register();
2190   Register length  = op->length()->as_register();
2191   Register tmp = op->tmp()->as_register();
2192 
2193   CodeStub* stub = op->stub();
2194   int flags = op->flags();
2195   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2196   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2197 
2198   // if we don't know anything, just go through the generic arraycopy
2199   if (default_type == nullptr // || basic_type == T_OBJECT
2200       ) {
2201     Label done;
2202     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2203 
2204     // Save the arguments in case the generic arraycopy fails and we
2205     // have to fall back to the JNI stub
2206     __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2207     __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2208     __ str(src,              Address(sp, 4*BytesPerWord));
2209 
2210     address copyfunc_addr = StubRoutines::generic_arraycopy();
2211     assert(copyfunc_addr != nullptr, "generic arraycopy stub required");
2212 
2213     // The arguments are in java calling convention so we shift them
2214     // to C convention
2215     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2216     __ mov(c_rarg0, j_rarg0);
2217     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2218     __ mov(c_rarg1, j_rarg1);
2219     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2220     __ mov(c_rarg2, j_rarg2);
2221     assert_different_registers(c_rarg3, j_rarg4);
2222     __ mov(c_rarg3, j_rarg3);
2223     __ mov(c_rarg4, j_rarg4);
2224 #ifndef PRODUCT
2225     if (PrintC1Statistics) {
2226       __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2227     }
2228 #endif
2229     __ far_call(RuntimeAddress(copyfunc_addr));
2230 
2231     __ cbz(r0, *stub->continuation());
2232 
2233     // Reload values from the stack so they are where the stub
2234     // expects them.
2235     __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2236     __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2237     __ ldr(src,              Address(sp, 4*BytesPerWord));
2238 
2239     // r0 is -1^K where K == partial copied count
2240     __ eonw(rscratch1, r0, zr);
2241     // adjust length down and src/end pos up by partial copied count
2242     __ subw(length, length, rscratch1);
2243     __ addw(src_pos, src_pos, rscratch1);
2244     __ addw(dst_pos, dst_pos, rscratch1);
2245     __ b(*stub->entry());
2246 
2247     __ bind(*stub->continuation());
2248     return;
2249   }
2250 
2251   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2252 
2253   int elem_size = type2aelembytes(basic_type);
2254   int scale = exact_log2(elem_size);
2255 
2256   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2257   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2258   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2259   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2260 
2261   // test for null
2262   if (flags & LIR_OpArrayCopy::src_null_check) {
2263     __ cbz(src, *stub->entry());
2264   }
2265   if (flags & LIR_OpArrayCopy::dst_null_check) {
2266     __ cbz(dst, *stub->entry());
2267   }
2268 
2269   // If the compiler was not able to prove that exact type of the source or the destination
2270   // of the arraycopy is an array type, check at runtime if the source or the destination is
2271   // an instance type.
2272   if (flags & LIR_OpArrayCopy::type_check) {
2273     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2274       __ load_klass(tmp, dst);
2275       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2276       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2277       __ br(Assembler::GE, *stub->entry());
2278     }
2279 
2280     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2281       __ load_klass(tmp, src);
2282       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2283       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2284       __ br(Assembler::GE, *stub->entry());
2285     }
2286   }
2287 
2288   // check if negative
2289   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2290     __ cmpw(src_pos, 0);
2291     __ br(Assembler::LT, *stub->entry());
2292   }
2293   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2294     __ cmpw(dst_pos, 0);
2295     __ br(Assembler::LT, *stub->entry());
2296   }
2297 
2298   if (flags & LIR_OpArrayCopy::length_positive_check) {
2299     __ cmpw(length, 0);
2300     __ br(Assembler::LT, *stub->entry());
2301   }
2302 
2303   if (flags & LIR_OpArrayCopy::src_range_check) {
2304     __ addw(tmp, src_pos, length);
2305     __ ldrw(rscratch1, src_length_addr);
2306     __ cmpw(tmp, rscratch1);
2307     __ br(Assembler::HI, *stub->entry());
2308   }
2309   if (flags & LIR_OpArrayCopy::dst_range_check) {
2310     __ addw(tmp, dst_pos, length);
2311     __ ldrw(rscratch1, dst_length_addr);
2312     __ cmpw(tmp, rscratch1);
2313     __ br(Assembler::HI, *stub->entry());
2314   }
2315 
2316   if (flags & LIR_OpArrayCopy::type_check) {
2317     // We don't know the array types are compatible
2318     if (basic_type != T_OBJECT) {
2319       // Simple test for basic type arrays
2320       if (UseCompressedClassPointers) {
2321         __ ldrw(tmp, src_klass_addr);
2322         __ ldrw(rscratch1, dst_klass_addr);
2323         __ cmpw(tmp, rscratch1);
2324       } else {
2325         __ ldr(tmp, src_klass_addr);
2326         __ ldr(rscratch1, dst_klass_addr);
2327         __ cmp(tmp, rscratch1);
2328       }
2329       __ br(Assembler::NE, *stub->entry());
2330     } else {
2331       // For object arrays, if src is a sub class of dst then we can
2332       // safely do the copy.
2333       Label cont, slow;
2334 
2335 #define PUSH(r1, r2)                                    \
2336       stp(r1, r2, __ pre(sp, -2 * wordSize));
2337 
2338 #define POP(r1, r2)                                     \
2339       ldp(r1, r2, __ post(sp, 2 * wordSize));
2340 
2341       __ PUSH(src, dst);
2342 
2343       __ load_klass(src, src);
2344       __ load_klass(dst, dst);
2345 
2346       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, nullptr);
2347 
2348       __ PUSH(src, dst);
2349       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2350       __ POP(src, dst);
2351 
2352       __ cbnz(src, cont);
2353 
2354       __ bind(slow);
2355       __ POP(src, dst);
2356 
2357       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2358       if (copyfunc_addr != nullptr) { // use stub if available
2359         // src is not a sub class of dst so we have to do a
2360         // per-element check.
2361 
2362         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2363         if ((flags & mask) != mask) {
2364           // Check that at least both of them object arrays.
2365           assert(flags & mask, "one of the two should be known to be an object array");
2366 
2367           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2368             __ load_klass(tmp, src);
2369           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2370             __ load_klass(tmp, dst);
2371           }
2372           int lh_offset = in_bytes(Klass::layout_helper_offset());
2373           Address klass_lh_addr(tmp, lh_offset);
2374           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2375           __ ldrw(rscratch1, klass_lh_addr);
2376           __ mov(rscratch2, objArray_lh);
2377           __ eorw(rscratch1, rscratch1, rscratch2);
2378           __ cbnzw(rscratch1, *stub->entry());
2379         }
2380 
2381        // Spill because stubs can use any register they like and it's
2382        // easier to restore just those that we care about.
2383         __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2384         __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2385         __ str(src,              Address(sp, 4*BytesPerWord));
2386 
2387         __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2388         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2389         assert_different_registers(c_rarg0, dst, dst_pos, length);
2390         __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2391         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2392         assert_different_registers(c_rarg1, dst, length);
2393         __ uxtw(c_rarg2, length);
2394         assert_different_registers(c_rarg2, dst);
2395 
2396         __ load_klass(c_rarg4, dst);
2397         __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2398         __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2399         __ far_call(RuntimeAddress(copyfunc_addr));
2400 
2401 #ifndef PRODUCT
2402         if (PrintC1Statistics) {
2403           Label failed;
2404           __ cbnz(r0, failed);
2405           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2406           __ bind(failed);
2407         }
2408 #endif
2409 
2410         __ cbz(r0, *stub->continuation());
2411 
2412 #ifndef PRODUCT
2413         if (PrintC1Statistics) {
2414           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2415         }
2416 #endif
2417         assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2418 
2419         // Restore previously spilled arguments
2420         __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2421         __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2422         __ ldr(src,              Address(sp, 4*BytesPerWord));
2423 
2424         // return value is -1^K where K is partial copied count
2425         __ eonw(rscratch1, r0, zr);
2426         // adjust length down and src/end pos up by partial copied count
2427         __ subw(length, length, rscratch1);
2428         __ addw(src_pos, src_pos, rscratch1);
2429         __ addw(dst_pos, dst_pos, rscratch1);
2430       }
2431 
2432       __ b(*stub->entry());
2433 
2434       __ bind(cont);
2435       __ POP(src, dst);
2436     }
2437   }
2438 
2439 #ifdef ASSERT
2440   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2441     // Sanity check the known type with the incoming class.  For the
2442     // primitive case the types must match exactly with src.klass and
2443     // dst.klass each exactly matching the default type.  For the
2444     // object array case, if no type check is needed then either the
2445     // dst type is exactly the expected type and the src type is a
2446     // subtype which we can't check or src is the same array as dst
2447     // but not necessarily exactly of type default_type.
2448     Label known_ok, halt;
2449     __ mov_metadata(tmp, default_type->constant_encoding());
2450     if (UseCompressedClassPointers) {
2451       __ encode_klass_not_null(tmp);
2452     }
2453 
2454     if (basic_type != T_OBJECT) {
2455 
2456       if (UseCompressedClassPointers) {
2457         __ ldrw(rscratch1, dst_klass_addr);
2458         __ cmpw(tmp, rscratch1);
2459       } else {
2460         __ ldr(rscratch1, dst_klass_addr);
2461         __ cmp(tmp, rscratch1);
2462       }
2463       __ br(Assembler::NE, halt);
2464       if (UseCompressedClassPointers) {
2465         __ ldrw(rscratch1, src_klass_addr);
2466         __ cmpw(tmp, rscratch1);
2467       } else {
2468         __ ldr(rscratch1, src_klass_addr);
2469         __ cmp(tmp, rscratch1);
2470       }
2471       __ br(Assembler::EQ, known_ok);
2472     } else {
2473       if (UseCompressedClassPointers) {
2474         __ ldrw(rscratch1, dst_klass_addr);
2475         __ cmpw(tmp, rscratch1);
2476       } else {
2477         __ ldr(rscratch1, dst_klass_addr);
2478         __ cmp(tmp, rscratch1);
2479       }
2480       __ br(Assembler::EQ, known_ok);
2481       __ cmp(src, dst);
2482       __ br(Assembler::EQ, known_ok);
2483     }
2484     __ bind(halt);
2485     __ stop("incorrect type information in arraycopy");
2486     __ bind(known_ok);
2487   }
2488 #endif
2489 
2490 #ifndef PRODUCT
2491   if (PrintC1Statistics) {
2492     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2493   }
2494 #endif
2495 
2496   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2497   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2498   assert_different_registers(c_rarg0, dst, dst_pos, length);
2499   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2500   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2501   assert_different_registers(c_rarg1, dst, length);
2502   __ uxtw(c_rarg2, length);
2503   assert_different_registers(c_rarg2, dst);
2504 
2505   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2506   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2507   const char *name;
2508   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2509 
2510  CodeBlob *cb = CodeCache::find_blob(entry);
2511  if (cb) {
2512    __ far_call(RuntimeAddress(entry));
2513  } else {
2514    __ call_VM_leaf(entry, 3);
2515  }
2516 
2517   __ bind(*stub->continuation());
2518 }
2519 
2520 
2521 
2522 
2523 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2524   Register obj = op->obj_opr()->as_register();  // may not be an oop
2525   Register hdr = op->hdr_opr()->as_register();
2526   Register lock = op->lock_opr()->as_register();
2527   Register temp = op->scratch_opr()->as_register();
2528   if (LockingMode == LM_MONITOR) {
2529     if (op->info() != nullptr) {
2530       add_debug_info_for_null_check_here(op->info());
2531       __ null_check(obj, -1);
2532     }
2533     __ b(*op->stub()->entry());
2534   } else if (op->code() == lir_lock) {
2535     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2536     // add debug info for NullPointerException only if one is possible
2537     int null_check_offset = __ lock_object(hdr, obj, lock, temp, *op->stub()->entry());
2538     if (op->info() != nullptr) {
2539       add_debug_info_for_null_check(null_check_offset, op->info());
2540     }
2541     // done
2542   } else if (op->code() == lir_unlock) {
2543     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2544     __ unlock_object(hdr, obj, lock, temp, *op->stub()->entry());
2545   } else {
2546     Unimplemented();
2547   }
2548   __ bind(*op->stub()->continuation());
2549 }
2550 
2551 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) {
2552   Register obj = op->obj()->as_pointer_register();
2553   Register result = op->result_opr()->as_pointer_register();
2554 
2555   CodeEmitInfo* info = op->info();
2556   if (info != nullptr) {
2557     add_debug_info_for_null_check_here(info);
2558   }
2559 
2560   if (UseCompressedClassPointers) {
2561     __ ldrw(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2562     __ decode_klass_not_null(result);
2563   } else {
2564     __ ldr(result, Address (obj, oopDesc::klass_offset_in_bytes()));
2565   }
2566 }
2567 
2568 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2569   ciMethod* method = op->profiled_method();
2570   int bci          = op->profiled_bci();
2571   ciMethod* callee = op->profiled_callee();
2572 
2573   // Update counter for all call types
2574   ciMethodData* md = method->method_data_or_null();
2575   assert(md != nullptr, "Sanity");
2576   ciProfileData* data = md->bci_to_data(bci);
2577   assert(data != nullptr && data->is_CounterData(), "need CounterData for calls");
2578   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2579   Register mdo  = op->mdo()->as_register();
2580   __ mov_metadata(mdo, md->constant_encoding());
2581   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2582   // Perform additional virtual call profiling for invokevirtual and
2583   // invokeinterface bytecodes
2584   if (op->should_profile_receiver_type()) {
2585     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2586     Register recv = op->recv()->as_register();
2587     assert_different_registers(mdo, recv);
2588     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2589     ciKlass* known_klass = op->known_holder();
2590     if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) {
2591       // We know the type that will be seen at this call site; we can
2592       // statically update the MethodData* rather than needing to do
2593       // dynamic tests on the receiver type
2594 
2595       // NOTE: we should probably put a lock around this search to
2596       // avoid collisions by concurrent compilations
2597       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2598       uint i;
2599       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2600         ciKlass* receiver = vc_data->receiver(i);
2601         if (known_klass->equals(receiver)) {
2602           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2603           __ addptr(data_addr, DataLayout::counter_increment);
2604           return;
2605         }
2606       }
2607 
2608       // Receiver type not found in profile data; select an empty slot
2609 
2610       // Note that this is less efficient than it should be because it
2611       // always does a write to the receiver part of the
2612       // VirtualCallData rather than just the first time
2613       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2614         ciKlass* receiver = vc_data->receiver(i);
2615         if (receiver == nullptr) {
2616           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2617           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2618           __ lea(rscratch2, recv_addr);
2619           __ str(rscratch1, Address(rscratch2));
2620           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2621           __ addptr(data_addr, DataLayout::counter_increment);
2622           return;
2623         }
2624       }
2625     } else {
2626       __ load_klass(recv, recv);
2627       Label update_done;
2628       type_profile_helper(mdo, md, data, recv, &update_done);
2629       // Receiver did not match any saved receiver and there is no empty row for it.
2630       // Increment total counter to indicate polymorphic case.
2631       __ addptr(counter_addr, DataLayout::counter_increment);
2632 
2633       __ bind(update_done);
2634     }
2635   } else {
2636     // Static call
2637     __ addptr(counter_addr, DataLayout::counter_increment);
2638   }
2639 }
2640 
2641 
2642 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2643   Unimplemented();
2644 }
2645 
2646 
2647 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2648   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2649 }
2650 
2651 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2652   assert(op->crc()->is_single_cpu(),  "crc must be register");
2653   assert(op->val()->is_single_cpu(),  "byte value must be register");
2654   assert(op->result_opr()->is_single_cpu(), "result must be register");
2655   Register crc = op->crc()->as_register();
2656   Register val = op->val()->as_register();
2657   Register res = op->result_opr()->as_register();
2658 
2659   assert_different_registers(val, crc, res);
2660   uint64_t offset;
2661   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2662   __ add(res, res, offset);
2663 
2664   __ mvnw(crc, crc); // ~crc
2665   __ update_byte_crc32(crc, val, res);
2666   __ mvnw(res, crc); // ~crc
2667 }
2668 
2669 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2670   COMMENT("emit_profile_type {");
2671   Register obj = op->obj()->as_register();
2672   Register tmp = op->tmp()->as_pointer_register();
2673   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2674   ciKlass* exact_klass = op->exact_klass();
2675   intptr_t current_klass = op->current_klass();
2676   bool not_null = op->not_null();
2677   bool no_conflict = op->no_conflict();
2678 
2679   Label update, next, none;
2680 
2681   bool do_null = !not_null;
2682   bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2683   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2684 
2685   assert(do_null || do_update, "why are we here?");
2686   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2687   assert(mdo_addr.base() != rscratch1, "wrong register");
2688 
2689   __ verify_oop(obj);
2690 
2691   if (tmp != obj) {
2692     assert_different_registers(obj, tmp, rscratch1, rscratch2, mdo_addr.base(), mdo_addr.index());
2693     __ mov(tmp, obj);
2694   } else {
2695     assert_different_registers(obj, rscratch1, rscratch2, mdo_addr.base(), mdo_addr.index());
2696   }
2697   if (do_null) {
2698     __ cbnz(tmp, update);
2699     if (!TypeEntries::was_null_seen(current_klass)) {
2700       __ ldr(rscratch2, mdo_addr);
2701       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2702       __ str(rscratch2, mdo_addr);
2703     }
2704     if (do_update) {
2705 #ifndef ASSERT
2706       __ b(next);
2707     }
2708 #else
2709       __ b(next);
2710     }
2711   } else {
2712     __ cbnz(tmp, update);
2713     __ stop("unexpected null obj");
2714 #endif
2715   }
2716 
2717   __ bind(update);
2718 
2719   if (do_update) {
2720 #ifdef ASSERT
2721     if (exact_klass != nullptr) {
2722       Label ok;
2723       __ load_klass(tmp, tmp);
2724       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2725       __ eor(rscratch1, tmp, rscratch1);
2726       __ cbz(rscratch1, ok);
2727       __ stop("exact klass and actual klass differ");
2728       __ bind(ok);
2729     }
2730 #endif
2731     if (!no_conflict) {
2732       if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) {
2733         if (exact_klass != nullptr) {
2734           __ mov_metadata(tmp, exact_klass->constant_encoding());
2735         } else {
2736           __ load_klass(tmp, tmp);
2737         }
2738 
2739         __ ldr(rscratch2, mdo_addr);
2740         __ eor(tmp, tmp, rscratch2);
2741         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2742         // klass seen before, nothing to do. The unknown bit may have been
2743         // set already but no need to check.
2744         __ cbz(rscratch1, next);
2745 
2746         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2747 
2748         if (TypeEntries::is_type_none(current_klass)) {
2749           __ cbz(rscratch2, none);
2750           __ cmp(rscratch2, (u1)TypeEntries::null_seen);
2751           __ br(Assembler::EQ, none);
2752           // There is a chance that the checks above
2753           // fail if another thread has just set the
2754           // profiling to this obj's klass
2755           __ dmb(Assembler::ISHLD);
2756           __ eor(tmp, tmp, rscratch2); // get back original value before XOR
2757           __ ldr(rscratch2, mdo_addr);
2758           __ eor(tmp, tmp, rscratch2);
2759           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2760           __ cbz(rscratch1, next);
2761         }
2762       } else {
2763         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2764                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2765 
2766         __ ldr(tmp, mdo_addr);
2767         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2768       }
2769 
2770       // different than before. Cannot keep accurate profile.
2771       __ ldr(rscratch2, mdo_addr);
2772       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2773       __ str(rscratch2, mdo_addr);
2774 
2775       if (TypeEntries::is_type_none(current_klass)) {
2776         __ b(next);
2777 
2778         __ bind(none);
2779         // first time here. Set profile type.
2780         __ str(tmp, mdo_addr);
2781 #ifdef ASSERT
2782         __ andr(tmp, tmp, TypeEntries::type_mask);
2783         __ verify_klass_ptr(tmp);
2784 #endif
2785       }
2786     } else {
2787       // There's a single possible klass at this profile point
2788       assert(exact_klass != nullptr, "should be");
2789       if (TypeEntries::is_type_none(current_klass)) {
2790         __ mov_metadata(tmp, exact_klass->constant_encoding());
2791         __ ldr(rscratch2, mdo_addr);
2792         __ eor(tmp, tmp, rscratch2);
2793         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2794         __ cbz(rscratch1, next);
2795 #ifdef ASSERT
2796         {
2797           Label ok;
2798           __ ldr(rscratch1, mdo_addr);
2799           __ cbz(rscratch1, ok);
2800           __ cmp(rscratch1, (u1)TypeEntries::null_seen);
2801           __ br(Assembler::EQ, ok);
2802           // may have been set by another thread
2803           __ dmb(Assembler::ISHLD);
2804           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2805           __ ldr(rscratch2, mdo_addr);
2806           __ eor(rscratch2, rscratch1, rscratch2);
2807           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2808           __ cbz(rscratch2, ok);
2809 
2810           __ stop("unexpected profiling mismatch");
2811           __ bind(ok);
2812         }
2813 #endif
2814         // first time here. Set profile type.
2815         __ str(tmp, mdo_addr);
2816 #ifdef ASSERT
2817         __ andr(tmp, tmp, TypeEntries::type_mask);
2818         __ verify_klass_ptr(tmp);
2819 #endif
2820       } else {
2821         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
2822                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2823 
2824         __ ldr(tmp, mdo_addr);
2825         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2826 
2827         __ orr(tmp, tmp, TypeEntries::type_unknown);
2828         __ str(tmp, mdo_addr);
2829         // FIXME: Write barrier needed here?
2830       }
2831     }
2832 
2833     __ bind(next);
2834   }
2835   COMMENT("} emit_profile_type");
2836 }
2837 
2838 
2839 void LIR_Assembler::align_backward_branch_target() {
2840 }
2841 
2842 
2843 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2844   // tmp must be unused
2845   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2846 
2847   if (left->is_single_cpu()) {
2848     assert(dest->is_single_cpu(), "expect single result reg");
2849     __ negw(dest->as_register(), left->as_register());
2850   } else if (left->is_double_cpu()) {
2851     assert(dest->is_double_cpu(), "expect double result reg");
2852     __ neg(dest->as_register_lo(), left->as_register_lo());
2853   } else if (left->is_single_fpu()) {
2854     assert(dest->is_single_fpu(), "expect single float result reg");
2855     __ fnegs(dest->as_float_reg(), left->as_float_reg());
2856   } else {
2857     assert(left->is_double_fpu(), "expect double float operand reg");
2858     assert(dest->is_double_fpu(), "expect double float result reg");
2859     __ fnegd(dest->as_double_reg(), left->as_double_reg());
2860   }
2861 }
2862 
2863 
2864 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2865   if (patch_code != lir_patch_none) {
2866     deoptimize_trap(info);
2867     return;
2868   }
2869 
2870   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2871 }
2872 
2873 
2874 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2875   assert(!tmp->is_valid(), "don't need temporary");
2876 
2877   CodeBlob *cb = CodeCache::find_blob(dest);
2878   if (cb) {
2879     __ far_call(RuntimeAddress(dest));
2880   } else {
2881     __ mov(rscratch1, RuntimeAddress(dest));
2882     __ blr(rscratch1);
2883   }
2884 
2885   if (info != nullptr) {
2886     add_call_info_here(info);
2887   }
2888   __ post_call_nop();
2889 }
2890 
2891 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2892   if (dest->is_address() || src->is_address()) {
2893     move_op(src, dest, type, lir_patch_none, info,
2894             /*pop_fpu_stack*/false, /*wide*/false);
2895   } else {
2896     ShouldNotReachHere();
2897   }
2898 }
2899 
2900 #ifdef ASSERT
2901 // emit run-time assertion
2902 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2903   assert(op->code() == lir_assert, "must be");
2904 
2905   if (op->in_opr1()->is_valid()) {
2906     assert(op->in_opr2()->is_valid(), "both operands must be valid");
2907     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2908   } else {
2909     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2910     assert(op->condition() == lir_cond_always, "no other conditions allowed");
2911   }
2912 
2913   Label ok;
2914   if (op->condition() != lir_cond_always) {
2915     Assembler::Condition acond = Assembler::AL;
2916     switch (op->condition()) {
2917       case lir_cond_equal:        acond = Assembler::EQ;  break;
2918       case lir_cond_notEqual:     acond = Assembler::NE;  break;
2919       case lir_cond_less:         acond = Assembler::LT;  break;
2920       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
2921       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
2922       case lir_cond_greater:      acond = Assembler::GT;  break;
2923       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
2924       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
2925       default:                    ShouldNotReachHere();
2926     }
2927     __ br(acond, ok);
2928   }
2929   if (op->halt()) {
2930     const char* str = __ code_string(op->msg());
2931     __ stop(str);
2932   } else {
2933     breakpoint();
2934   }
2935   __ bind(ok);
2936 }
2937 #endif
2938 
2939 #ifndef PRODUCT
2940 #define COMMENT(x)   do { __ block_comment(x); } while (0)
2941 #else
2942 #define COMMENT(x)
2943 #endif
2944 
2945 void LIR_Assembler::membar() {
2946   COMMENT("membar");
2947   __ membar(MacroAssembler::AnyAny);
2948 }
2949 
2950 void LIR_Assembler::membar_acquire() {
2951   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2952 }
2953 
2954 void LIR_Assembler::membar_release() {
2955   __ membar(Assembler::LoadStore|Assembler::StoreStore);
2956 }
2957 
2958 void LIR_Assembler::membar_loadload() {
2959   __ membar(Assembler::LoadLoad);
2960 }
2961 
2962 void LIR_Assembler::membar_storestore() {
2963   __ membar(MacroAssembler::StoreStore);
2964 }
2965 
2966 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2967 
2968 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2969 
2970 void LIR_Assembler::on_spin_wait() {
2971   __ spin_wait();
2972 }
2973 
2974 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2975   __ mov(result_reg->as_register(), rthread);
2976 }
2977 
2978 
2979 void LIR_Assembler::peephole(LIR_List *lir) {
2980 #if 0
2981   if (tableswitch_count >= max_tableswitches)
2982     return;
2983 
2984   /*
2985     This finite-state automaton recognizes sequences of compare-and-
2986     branch instructions.  We will turn them into a tableswitch.  You
2987     could argue that C1 really shouldn't be doing this sort of
2988     optimization, but without it the code is really horrible.
2989   */
2990 
2991   enum { start_s, cmp1_s, beq_s, cmp_s } state;
2992   int first_key, last_key = -2147483648;
2993   int next_key = 0;
2994   int start_insn = -1;
2995   int last_insn = -1;
2996   Register reg = noreg;
2997   LIR_Opr reg_opr;
2998   state = start_s;
2999 
3000   LIR_OpList* inst = lir->instructions_list();
3001   for (int i = 0; i < inst->length(); i++) {
3002     LIR_Op* op = inst->at(i);
3003     switch (state) {
3004     case start_s:
3005       first_key = -1;
3006       start_insn = i;
3007       switch (op->code()) {
3008       case lir_cmp:
3009         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3010         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3011         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3012             && opr2->is_constant()
3013             && opr2->type() == T_INT) {
3014           reg_opr = opr1;
3015           reg = opr1->as_register();
3016           first_key = opr2->as_constant_ptr()->as_jint();
3017           next_key = first_key + 1;
3018           state = cmp_s;
3019           goto next_state;
3020         }
3021         break;
3022       }
3023       break;
3024     case cmp_s:
3025       switch (op->code()) {
3026       case lir_branch:
3027         if (op->as_OpBranch()->cond() == lir_cond_equal) {
3028           state = beq_s;
3029           last_insn = i;
3030           goto next_state;
3031         }
3032       }
3033       state = start_s;
3034       break;
3035     case beq_s:
3036       switch (op->code()) {
3037       case lir_cmp: {
3038         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3039         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3040         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3041             && opr1->as_register() == reg
3042             && opr2->is_constant()
3043             && opr2->type() == T_INT
3044             && opr2->as_constant_ptr()->as_jint() == next_key) {
3045           last_key = next_key;
3046           next_key++;
3047           state = cmp_s;
3048           goto next_state;
3049         }
3050       }
3051       }
3052       last_key = next_key;
3053       state = start_s;
3054       break;
3055     default:
3056       assert(false, "impossible state");
3057     }
3058     if (state == start_s) {
3059       if (first_key < last_key - 5L && reg != noreg) {
3060         {
3061           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3062           //        reg->encoding(),
3063           //        start_insn, first_key, last_key);
3064           //   for (int i = 0; i < inst->length(); i++) {
3065           //     inst->at(i)->print();
3066           //     tty->print("\n");
3067           //   }
3068           //   tty->print("\n");
3069         }
3070 
3071         struct tableswitch *sw = &switches[tableswitch_count];
3072         sw->_insn_index = start_insn, sw->_first_key = first_key,
3073           sw->_last_key = last_key, sw->_reg = reg;
3074         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3075         {
3076           // Insert the new table of branches
3077           int offset = last_insn;
3078           for (int n = first_key; n < last_key; n++) {
3079             inst->insert_before
3080               (last_insn + 1,
3081                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3082                                 inst->at(offset)->as_OpBranch()->label()));
3083             offset -= 2, i++;
3084           }
3085         }
3086         // Delete all the old compare-and-branch instructions
3087         for (int n = first_key; n < last_key; n++) {
3088           inst->remove_at(start_insn);
3089           inst->remove_at(start_insn);
3090         }
3091         // Insert the tableswitch instruction
3092         inst->insert_before(start_insn,
3093                             new LIR_Op2(lir_cmp, lir_cond_always,
3094                                         LIR_OprFact::intConst(tableswitch_count),
3095                                         reg_opr));
3096         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3097         tableswitch_count++;
3098       }
3099       reg = noreg;
3100       last_key = -2147483648;
3101     }
3102   next_state:
3103     ;
3104   }
3105 #endif
3106 }
3107 
3108 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3109   Address addr = as_Address(src->as_address_ptr());
3110   BasicType type = src->type();
3111   bool is_oop = is_reference_type(type);
3112 
3113   void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3114   void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3115 
3116   switch(type) {
3117   case T_INT:
3118     xchg = &MacroAssembler::atomic_xchgalw;
3119     add = &MacroAssembler::atomic_addalw;
3120     break;
3121   case T_LONG:
3122     xchg = &MacroAssembler::atomic_xchgal;
3123     add = &MacroAssembler::atomic_addal;
3124     break;
3125   case T_OBJECT:
3126   case T_ARRAY:
3127     if (UseCompressedOops) {
3128       xchg = &MacroAssembler::atomic_xchgalw;
3129       add = &MacroAssembler::atomic_addalw;
3130     } else {
3131       xchg = &MacroAssembler::atomic_xchgal;
3132       add = &MacroAssembler::atomic_addal;
3133     }
3134     break;
3135   default:
3136     ShouldNotReachHere();
3137     xchg = &MacroAssembler::atomic_xchgal;
3138     add = &MacroAssembler::atomic_addal; // unreachable
3139   }
3140 
3141   switch (code) {
3142   case lir_xadd:
3143     {
3144       RegisterOrConstant inc;
3145       Register tmp = as_reg(tmp_op);
3146       Register dst = as_reg(dest);
3147       if (data->is_constant()) {
3148         inc = RegisterOrConstant(as_long(data));
3149         assert_different_registers(dst, addr.base(), tmp,
3150                                    rscratch1, rscratch2);
3151       } else {
3152         inc = RegisterOrConstant(as_reg(data));
3153         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3154                                    rscratch1, rscratch2);
3155       }
3156       __ lea(tmp, addr);
3157       (_masm->*add)(dst, inc, tmp);
3158       break;
3159     }
3160   case lir_xchg:
3161     {
3162       Register tmp = tmp_op->as_register();
3163       Register obj = as_reg(data);
3164       Register dst = as_reg(dest);
3165       if (is_oop && UseCompressedOops) {
3166         __ encode_heap_oop(rscratch2, obj);
3167         obj = rscratch2;
3168       }
3169       assert_different_registers(obj, addr.base(), tmp, rscratch1);
3170       assert_different_registers(dst, addr.base(), tmp, rscratch1);
3171       __ lea(tmp, addr);
3172       (_masm->*xchg)(dst, obj, tmp);
3173       if (is_oop && UseCompressedOops) {
3174         __ decode_heap_oop(dst);
3175       }
3176     }
3177     break;
3178   default:
3179     ShouldNotReachHere();
3180   }
3181   __ membar(__ AnyAny);
3182 }
3183 
3184 #undef __