1 /*
   2  * Copyright (c) 2013, Red Hat Inc.
   3  * Copyright (c) 2000, 2020, Oracle and/or its affiliates.
   4  * All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Compilation.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_MacroAssembler.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArrayKlass.hpp"
  36 #include "ci/ciInstance.hpp"
  37 #include "gc_interface/collectedHeap.hpp"
  38 #include "memory/barrierSet.hpp"
  39 #include "memory/cardTableModRefBS.hpp"
  40 #include "nativeInst_aarch64.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "vmreg_aarch64.inline.hpp"
  44 
  45 
  46 #if INCLUDE_ALL_GCS
  47 #include "shenandoahBarrierSetAssembler_aarch64.hpp"
  48 #endif
  49 
  50 #ifndef PRODUCT
  51 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  52 #else
  53 #define COMMENT(x)
  54 #endif
  55 
  56 NEEDS_CLEANUP // remove this definitions ?
  57 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  58 const Register SYNC_header = r0;   // synchronization header
  59 const Register SHIFT_count = r0;   // where count for shift operations must be
  60 
  61 #define __ _masm->
  62 
  63 
  64 static void select_different_registers(Register preserve,
  65                                        Register extra,
  66                                        Register &tmp1,
  67                                        Register &tmp2) {
  68   if (tmp1 == preserve) {
  69     assert_different_registers(tmp1, tmp2, extra);
  70     tmp1 = extra;
  71   } else if (tmp2 == preserve) {
  72     assert_different_registers(tmp1, tmp2, extra);
  73     tmp2 = extra;
  74   }
  75   assert_different_registers(preserve, tmp1, tmp2);
  76 }
  77 
  78 
  79 
  80 static void select_different_registers(Register preserve,
  81                                        Register extra,
  82                                        Register &tmp1,
  83                                        Register &tmp2,
  84                                        Register &tmp3) {
  85   if (tmp1 == preserve) {
  86     assert_different_registers(tmp1, tmp2, tmp3, extra);
  87     tmp1 = extra;
  88   } else if (tmp2 == preserve) {
  89     assert_different_registers(tmp1, tmp2, tmp3, extra);
  90     tmp2 = extra;
  91   } else if (tmp3 == preserve) {
  92     assert_different_registers(tmp1, tmp2, tmp3, extra);
  93     tmp3 = extra;
  94   }
  95   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  96 }
  97 
  98 
  99 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
 100 
 101 
 102 LIR_Opr LIR_Assembler::receiverOpr() {
 103   return FrameMap::receiver_opr;
 104 }
 105 
 106 LIR_Opr LIR_Assembler::osrBufferPointer() {
 107   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 108 }
 109 
 110 //--------------fpu register translations-----------------------
 111 
 112 
 113 address LIR_Assembler::float_constant(float f) {
 114   address const_addr = __ float_constant(f);
 115   if (const_addr == NULL) {
 116     bailout("const section overflow");
 117     return __ code()->consts()->start();
 118   } else {
 119     return const_addr;
 120   }
 121 }
 122 
 123 
 124 address LIR_Assembler::double_constant(double d) {
 125   address const_addr = __ double_constant(d);
 126   if (const_addr == NULL) {
 127     bailout("const section overflow");
 128     return __ code()->consts()->start();
 129   } else {
 130     return const_addr;
 131   }
 132 }
 133 
 134 address LIR_Assembler::int_constant(jlong n) {
 135   address const_addr = __ long_constant(n);
 136   if (const_addr == NULL) {
 137     bailout("const section overflow");
 138     return __ code()->consts()->start();
 139   } else {
 140     return const_addr;
 141   }
 142 }
 143 
 144 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
 145 
 146 void LIR_Assembler::reset_FPU() { Unimplemented(); }
 147 
 148 void LIR_Assembler::fpop() { Unimplemented(); }
 149 
 150 void LIR_Assembler::fxch(int i) { Unimplemented(); }
 151 
 152 void LIR_Assembler::fld(int i) { Unimplemented(); }
 153 
 154 void LIR_Assembler::ffree(int i) { Unimplemented(); }
 155 
 156 void LIR_Assembler::breakpoint() { Unimplemented(); }
 157 
 158 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 159 
 160 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 161 
 162 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
 163 //-------------------------------------------
 164 
 165 static Register as_reg(LIR_Opr op) {
 166   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 167 }
 168 
 169 static jlong as_long(LIR_Opr data) {
 170   jlong result;
 171   switch (data->type()) {
 172   case T_INT:
 173     result = (data->as_jint());
 174     break;
 175   case T_LONG:
 176     result = (data->as_jlong());
 177     break;
 178   default:
 179     ShouldNotReachHere();
 180   }
 181   return result;
 182 }
 183 
 184 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 185   Register base = addr->base()->as_pointer_register();
 186   LIR_Opr opr = addr->index();
 187   if (opr->is_cpu_register()) {
 188     Register index;
 189     if (opr->is_single_cpu())
 190       index = opr->as_register();
 191     else
 192       index = opr->as_register_lo();
 193     assert(addr->disp() == 0, "must be");
 194     switch(opr->type()) {
 195       case T_INT:
 196         return Address(base, index, Address::sxtw(addr->scale()));
 197       case T_LONG:
 198         return Address(base, index, Address::lsl(addr->scale()));
 199       default:
 200         ShouldNotReachHere();
 201       }
 202   } else  {
 203     intptr_t addr_offset = intptr_t(addr->disp());
 204     if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
 205       return Address(base, addr_offset, Address::lsl(addr->scale()));
 206     else {
 207       __ mov(tmp, addr_offset);
 208       return Address(base, tmp, Address::lsl(addr->scale()));
 209     }
 210   }
 211   return Address();
 212 }
 213 
 214 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 215   ShouldNotReachHere();
 216   return Address();
 217 }
 218 
 219 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 220   return as_Address(addr, rscratch1);
 221 }
 222 
 223 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 224   return as_Address(addr, rscratch1);  // Ouch
 225   // FIXME: This needs to be much more clever.  See x86.
 226 }
 227 
 228 
 229 void LIR_Assembler::osr_entry() {
 230   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 231   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 232   ValueStack* entry_state = osr_entry->state();
 233   int number_of_locks = entry_state->locks_size();
 234 
 235   // we jump here if osr happens with the interpreter
 236   // state set up to continue at the beginning of the
 237   // loop that triggered osr - in particular, we have
 238   // the following registers setup:
 239   //
 240   // r2: osr buffer
 241   //
 242 
 243   // build frame
 244   ciMethod* m = compilation()->method();
 245   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 246 
 247   // OSR buffer is
 248   //
 249   // locals[nlocals-1..0]
 250   // monitors[0..number_of_locks]
 251   //
 252   // locals is a direct copy of the interpreter frame so in the osr buffer
 253   // so first slot in the local array is the last local from the interpreter
 254   // and last slot is local[0] (receiver) from the interpreter
 255   //
 256   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 257   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 258   // in the interpreter frame (the method lock if a sync method)
 259 
 260   // Initialize monitors in the compiled activation.
 261   //   r2: pointer to osr buffer
 262   //
 263   // All other registers are dead at this point and the locals will be
 264   // copied into place by code emitted in the IR.
 265 
 266   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 267   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 268     int monitor_offset = BytesPerWord * method()->max_locals() +
 269       (2 * BytesPerWord) * (number_of_locks - 1);
 270     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 271     // the OSR buffer using 2 word entries: first the lock and then
 272     // the oop.
 273     for (int i = 0; i < number_of_locks; i++) {
 274       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 275 #ifdef ASSERT
 276       // verify the interpreter's monitor has a non-null object
 277       {
 278         Label L;
 279         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 280         __ cbnz(rscratch1, L);
 281         __ stop("locked object is NULL");
 282         __ bind(L);
 283       }
 284 #endif
 285       __ ldr(r19, Address(OSR_buf, slot_offset + 0));
 286       __ str(r19, frame_map()->address_for_monitor_lock(i));
 287       __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 288       __ str(r19, frame_map()->address_for_monitor_object(i));
 289     }
 290   }
 291 }
 292 
 293 
 294 // inline cache check; done before the frame is built.
 295 int LIR_Assembler::check_icache() {
 296   Register receiver = FrameMap::receiver_opr->as_register();
 297   Register ic_klass = IC_Klass;
 298   int start_offset = __ offset();
 299   __ inline_cache_check(receiver, ic_klass);
 300 
 301   // if icache check fails, then jump to runtime routine
 302   // Note: RECEIVER must still contain the receiver!
 303   Label dont;
 304   __ br(Assembler::EQ, dont);
 305   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 306 
 307   // We align the verified entry point unless the method body
 308   // (including its inline cache check) will fit in a single 64-byte
 309   // icache line.
 310   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 311     // force alignment after the cache check.
 312     __ align(CodeEntryAlignment);
 313   }
 314 
 315   __ bind(dont);
 316   return start_offset;
 317 }
 318 
 319 
 320 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 321   if (o == NULL) {
 322     __ mov(reg, zr);
 323   } else {
 324     __ movoop(reg, o, /*immediate*/true);
 325   }
 326 }
 327 
 328 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 329   address target = NULL;
 330   relocInfo::relocType reloc_type = relocInfo::none;
 331 
 332   switch (patching_id(info)) {
 333   case PatchingStub::access_field_id:
 334     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 335     reloc_type = relocInfo::section_word_type;
 336     break;
 337   case PatchingStub::load_klass_id:
 338     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 339     reloc_type = relocInfo::metadata_type;
 340     break;
 341   case PatchingStub::load_mirror_id:
 342     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 343     reloc_type = relocInfo::oop_type;
 344     break;
 345   case PatchingStub::load_appendix_id:
 346     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 347     reloc_type = relocInfo::oop_type;
 348     break;
 349   default: ShouldNotReachHere();
 350   }
 351 
 352   __ far_call(RuntimeAddress(target));
 353   add_call_info_here(info);
 354 }
 355 
 356 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 357   deoptimize_trap(info);
 358 }
 359 
 360 
 361 // This specifies the rsp decrement needed to build the frame
 362 int LIR_Assembler::initial_frame_size_in_bytes() const {
 363   // if rounding, must let FrameMap know!
 364 
 365   // The frame_map records size in slots (32bit word)
 366 
 367   // subtract two words to account for return address and link
 368   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 369 }
 370 
 371 
 372 int LIR_Assembler::emit_exception_handler() {
 373   // if the last instruction is a call (typically to do a throw which
 374   // is coming at the end after block reordering) the return address
 375   // must still point into the code area in order to avoid assertion
 376   // failures when searching for the corresponding bci => add a nop
 377   // (was bug 5/14/1999 - gri)
 378   __ nop();
 379 
 380   // generate code for exception handler
 381   address handler_base = __ start_a_stub(exception_handler_size);
 382   if (handler_base == NULL) {
 383     // not enough space left for the handler
 384     bailout("exception handler overflow");
 385     return -1;
 386   }
 387 
 388   int offset = code_offset();
 389 
 390   // the exception oop and pc are in r0, and r3
 391   // no other registers need to be preserved, so invalidate them
 392   __ invalidate_registers(false, true, true, false, true, true);
 393 
 394   // check that there is really an exception
 395   __ verify_not_null_oop(r0);
 396 
 397   // search an exception handler (r0: exception oop, r3: throwing pc)
 398   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));
 399   guarantee(code_offset() - offset <= exception_handler_size, "overflow");
 400   __ end_a_stub();
 401 
 402   return offset;
 403 }
 404 
 405 
 406 // Emit the code to remove the frame from the stack in the exception
 407 // unwind path.
 408 int LIR_Assembler::emit_unwind_handler() {
 409 #ifndef PRODUCT
 410   if (CommentedAssembly) {
 411     _masm->block_comment("Unwind handler");
 412   }
 413 #endif
 414 
 415   int offset = code_offset();
 416 
 417   // Fetch the exception from TLS and clear out exception related thread state
 418   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 419   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 420   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 421 
 422   __ bind(_unwind_handler_entry);
 423   __ verify_not_null_oop(r0);
 424   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 425     __ mov(r19, r0);  // Preserve the exception
 426   }
 427 
 428   // Preform needed unlocking
 429   MonitorExitStub* stub = NULL;
 430   if (method()->is_synchronized()) {
 431     monitor_address(0, FrameMap::r0_opr);
 432     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
 433     __ unlock_object(r5, r4, r0, *stub->entry());
 434     __ bind(*stub->continuation());
 435   }
 436 
 437   if (compilation()->env()->dtrace_method_probes()) {
 438     __ mov(c_rarg0, rthread);
 439     __ mov_metadata(c_rarg1, method()->constant_encoding());
 440     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1);
 441   }
 442 
 443   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 444     __ mov(r0, r19);  // Restore the exception
 445   }
 446 
 447   // remove the activation and dispatch to the unwind handler
 448   __ block_comment("remove_frame and dispatch to the unwind handler");
 449   __ remove_frame(initial_frame_size_in_bytes());
 450   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 451 
 452   // Emit the slow path assembly
 453   if (stub != NULL) {
 454     stub->emit_code(this);
 455   }
 456 
 457   return offset;
 458 }
 459 
 460 
 461 int LIR_Assembler::emit_deopt_handler() {
 462   // if the last instruction is a call (typically to do a throw which
 463   // is coming at the end after block reordering) the return address
 464   // must still point into the code area in order to avoid assertion
 465   // failures when searching for the corresponding bci => add a nop
 466   // (was bug 5/14/1999 - gri)
 467   __ nop();
 468 
 469   // generate code for exception handler
 470   address handler_base = __ start_a_stub(deopt_handler_size);
 471   if (handler_base == NULL) {
 472     // not enough space left for the handler
 473     bailout("deopt handler overflow");
 474     return -1;
 475   }
 476 
 477   int offset = code_offset();
 478 
 479   __ adr(lr, pc());
 480   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 481   guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
 482   __ end_a_stub();
 483 
 484   return offset;
 485 }
 486 
 487 
 488 // This is the fast version of java.lang.String.compare; it has not
 489 // OSR-entry and therefore, we generate a slow version for OSR's
 490 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info)  {
 491   __ mov(r2, (address)__FUNCTION__);
 492   __ call_Unimplemented();
 493 }
 494 
 495 
 496 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 497   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 498   int pc_offset = code_offset();
 499   flush_debug_info(pc_offset);
 500   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 501   if (info->exception_handlers() != NULL) {
 502     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 503   }
 504 }
 505 
 506 // Rather than take a segfault when the polling page is protected,
 507 // explicitly check for a safepoint in progress and if there is one,
 508 // fake a call to the handler as if a segfault had been caught.
 509 void LIR_Assembler::poll_for_safepoint(relocInfo::relocType rtype, CodeEmitInfo* info) {
 510   __ mov(rscratch1, SafepointSynchronize::address_of_state());
 511   __ ldrb(rscratch1, Address(rscratch1));
 512   Label nope, poll;
 513   __ cbz(rscratch1, nope);
 514   __ block_comment("safepoint");
 515   __ enter();
 516   __ push(0x3, sp);                // r0 & r1
 517   __ push(0x3ffffffc, sp);         // integer registers except lr & sp & r0 & r1
 518   __ adr(r0, poll);
 519   __ str(r0, Address(rthread, JavaThread::saved_exception_pc_offset()));
 520   __ mov(rscratch1, CAST_FROM_FN_PTR(address, SharedRuntime::get_poll_stub));
 521   __ blr(rscratch1);
 522   __ maybe_isb();
 523   __ pop(0x3ffffffc, sp);          // integer registers except lr & sp & r0 & r1
 524   __ mov(rscratch1, r0);
 525   __ pop(0x3, sp);                 // r0 & r1
 526   __ leave();
 527   __ br(rscratch1);
 528   address polling_page(os::get_polling_page());
 529   assert(os::is_poll_address(polling_page), "should be");
 530   unsigned long off;
 531   __ adrp(rscratch1, Address(polling_page, rtype), off);
 532   __ bind(poll);
 533   if (info)
 534     add_debug_info_for_branch(info);  // This isn't just debug info:
 535                                       // it's the oop map
 536   else
 537     __ code_section()->relocate(pc(), rtype);
 538   __ ldrw(zr, Address(rscratch1, off));
 539   __ bind(nope);
 540 }
 541 
 542 void LIR_Assembler::return_op(LIR_Opr result) {
 543   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 544   // Pop the stack before the safepoint code
 545   __ remove_frame(initial_frame_size_in_bytes());
 546   if (UseCompilerSafepoints) {
 547     address polling_page(os::get_polling_page());
 548     __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
 549   } else {
 550     poll_for_safepoint(relocInfo::poll_return_type);
 551   }
 552   __ ret(lr);
 553 }
 554 
 555 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 556   address polling_page(os::get_polling_page());
 557   if (UseCompilerSafepoints) {
 558     guarantee(info != NULL, "Shouldn't be NULL");
 559     assert(os::is_poll_address(polling_page), "should be");
 560     unsigned long off;
 561     __ adrp(rscratch1, Address(polling_page, relocInfo::poll_type), off);
 562     assert(off == 0, "must be");
 563     add_debug_info_for_branch(info);  // This isn't just debug info:
 564                                       // it's the oop map
 565     __ read_polling_page(rscratch1, relocInfo::poll_type);
 566   } else {
 567     poll_for_safepoint(relocInfo::poll_type, info);
 568   }
 569 
 570   return __ offset();
 571 }
 572 
 573 
 574 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 575   if (from_reg == r31_sp)
 576     from_reg = sp;
 577   if (to_reg == r31_sp)
 578     to_reg = sp;
 579   __ mov(to_reg, from_reg);
 580 }
 581 
 582 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 583 
 584 
 585 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 586   assert(src->is_constant(), "should not call otherwise");
 587   assert(dest->is_register(), "should not call otherwise");
 588   LIR_Const* c = src->as_constant_ptr();
 589 
 590   switch (c->type()) {
 591     case T_INT: {
 592       assert(patch_code == lir_patch_none, "no patching handled here");
 593       __ movw(dest->as_register(), c->as_jint());
 594       break;
 595     }
 596 
 597     case T_ADDRESS: {
 598       assert(patch_code == lir_patch_none, "no patching handled here");
 599       __ mov(dest->as_register(), c->as_jint());
 600       break;
 601     }
 602 
 603     case T_LONG: {
 604       assert(patch_code == lir_patch_none, "no patching handled here");
 605       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
 606       break;
 607     }
 608 
 609     case T_OBJECT: {
 610         if (patch_code == lir_patch_none) {
 611           jobject2reg(c->as_jobject(), dest->as_register());
 612         } else {
 613           jobject2reg_with_patching(dest->as_register(), info);
 614         }
 615       break;
 616     }
 617 
 618     case T_METADATA: {
 619       if (patch_code != lir_patch_none) {
 620         klass2reg_with_patching(dest->as_register(), info);
 621       } else {
 622         __ mov_metadata(dest->as_register(), c->as_metadata());
 623       }
 624       break;
 625     }
 626 
 627     case T_FLOAT: {
 628       if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 629         __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
 630       } else {
 631         __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 632         __ ldrs(dest->as_float_reg(), Address(rscratch1));
 633       }
 634       break;
 635     }
 636 
 637     case T_DOUBLE: {
 638       if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
 639         __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
 640       } else {
 641         __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 642         __ ldrd(dest->as_double_reg(), Address(rscratch1));
 643       }
 644       break;
 645     }
 646 
 647     default:
 648       ShouldNotReachHere();
 649   }
 650 }
 651 
 652 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 653   LIR_Const* c = src->as_constant_ptr();
 654   switch (c->type()) {
 655   case T_OBJECT:
 656     {
 657       if (! c->as_jobject())
 658         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 659       else {
 660         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 661         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 662       }
 663     }
 664     break;
 665   case T_ADDRESS:
 666     {
 667       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 668       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 669     }
 670   case T_INT:
 671   case T_FLOAT:
 672     {
 673       Register reg = zr;
 674       if (c->as_jint_bits() == 0)
 675         __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 676       else {
 677         __ movw(rscratch1, c->as_jint_bits());
 678         __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 679       }
 680     }
 681     break;
 682   case T_LONG:
 683   case T_DOUBLE:
 684     {
 685       Register reg = zr;
 686       if (c->as_jlong_bits() == 0)
 687         __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 688                                                  lo_word_offset_in_bytes));
 689       else {
 690         __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
 691         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 692                                                         lo_word_offset_in_bytes));
 693       }
 694     }
 695     break;
 696   default:
 697     ShouldNotReachHere();
 698   }
 699 }
 700 
 701 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 702   assert(src->is_constant(), "should not call otherwise");
 703   LIR_Const* c = src->as_constant_ptr();
 704   LIR_Address* to_addr = dest->as_address_ptr();
 705 
 706   void (Assembler::* insn)(Register Rt, const Address &adr);
 707 
 708   switch (type) {
 709   case T_ADDRESS:
 710     assert(c->as_jint() == 0, "should be");
 711     insn = &Assembler::str;
 712     break;
 713   case T_LONG:
 714     assert(c->as_jlong() == 0, "should be");
 715     insn = &Assembler::str;
 716     break;
 717   case T_INT:
 718     assert(c->as_jint() == 0, "should be");
 719     insn = &Assembler::strw;
 720     break;
 721   case T_OBJECT:
 722   case T_ARRAY:
 723     assert(c->as_jobject() == 0, "should be");
 724     if (UseCompressedOops && !wide) {
 725       insn = &Assembler::strw;
 726     } else {
 727       insn = &Assembler::str;
 728     }
 729     break;
 730   case T_CHAR:
 731   case T_SHORT:
 732     assert(c->as_jint() == 0, "should be");
 733     insn = &Assembler::strh;
 734     break;
 735   case T_BOOLEAN:
 736   case T_BYTE:
 737     assert(c->as_jint() == 0, "should be");
 738     insn = &Assembler::strb;
 739     break;
 740   default:
 741     ShouldNotReachHere();
 742   }
 743 
 744   if (info) add_debug_info_for_null_check_here(info);
 745   (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
 746 }
 747 
 748 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 749   assert(src->is_register(), "should not call otherwise");
 750   assert(dest->is_register(), "should not call otherwise");
 751 
 752   // move between cpu-registers
 753   if (dest->is_single_cpu()) {
 754     if (src->type() == T_LONG) {
 755       // Can do LONG -> OBJECT
 756       move_regs(src->as_register_lo(), dest->as_register());
 757       return;
 758     }
 759     assert(src->is_single_cpu(), "must match");
 760     if (src->type() == T_OBJECT) {
 761       __ verify_oop(src->as_register());
 762     }
 763     move_regs(src->as_register(), dest->as_register());
 764 
 765   } else if (dest->is_double_cpu()) {
 766     if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
 767       // Surprising to me but we can see move of a long to t_object
 768       __ verify_oop(src->as_register());
 769       move_regs(src->as_register(), dest->as_register_lo());
 770       return;
 771     }
 772     assert(src->is_double_cpu(), "must match");
 773     Register f_lo = src->as_register_lo();
 774     Register f_hi = src->as_register_hi();
 775     Register t_lo = dest->as_register_lo();
 776     Register t_hi = dest->as_register_hi();
 777     assert(f_hi == f_lo, "must be same");
 778     assert(t_hi == t_lo, "must be same");
 779     move_regs(f_lo, t_lo);
 780 
 781   } else if (dest->is_single_fpu()) {
 782     __ fmovs(dest->as_float_reg(), src->as_float_reg());
 783 
 784   } else if (dest->is_double_fpu()) {
 785     __ fmovd(dest->as_double_reg(), src->as_double_reg());
 786 
 787   } else {
 788     ShouldNotReachHere();
 789   }
 790 }
 791 
 792 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 793   if (src->is_single_cpu()) {
 794     if (type == T_ARRAY || type == T_OBJECT) {
 795       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 796       __ verify_oop(src->as_register());
 797     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 798       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 799     } else {
 800       __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 801     }
 802 
 803   } else if (src->is_double_cpu()) {
 804     Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 805     __ str(src->as_register_lo(), dest_addr_LO);
 806 
 807   } else if (src->is_single_fpu()) {
 808     Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 809     __ strs(src->as_float_reg(), dest_addr);
 810 
 811   } else if (src->is_double_fpu()) {
 812     Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 813     __ strd(src->as_double_reg(), dest_addr);
 814 
 815   } else {
 816     ShouldNotReachHere();
 817   }
 818 
 819 }
 820 
 821 
 822 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 823   LIR_Address* to_addr = dest->as_address_ptr();
 824   PatchingStub* patch = NULL;
 825   Register compressed_src = rscratch1;
 826 
 827   if (patch_code != lir_patch_none) {
 828     deoptimize_trap(info);
 829     return;
 830   }
 831 
 832   if (type == T_ARRAY || type == T_OBJECT) {
 833     __ verify_oop(src->as_register());
 834 
 835     if (UseCompressedOops && !wide) {
 836       __ encode_heap_oop(compressed_src, src->as_register());
 837     } else {
 838       compressed_src = src->as_register();
 839     }
 840   }
 841 
 842   int null_check_here = code_offset();
 843   switch (type) {
 844     case T_FLOAT: {
 845       __ strs(src->as_float_reg(), as_Address(to_addr));
 846       break;
 847     }
 848 
 849     case T_DOUBLE: {
 850       __ strd(src->as_double_reg(), as_Address(to_addr));
 851       break;
 852     }
 853 
 854     case T_ARRAY:   // fall through
 855     case T_OBJECT:  // fall through
 856       if (UseCompressedOops && !wide) {
 857         __ strw(compressed_src, as_Address(to_addr, rscratch2));
 858       } else {
 859          __ str(compressed_src, as_Address(to_addr));
 860       }
 861       break;
 862     case T_METADATA:
 863       // We get here to store a method pointer to the stack to pass to
 864       // a dtrace runtime call. This can't work on 64 bit with
 865       // compressed klass ptrs: T_METADATA can be a compressed klass
 866       // ptr or a 64 bit method pointer.
 867       LP64_ONLY(ShouldNotReachHere());
 868       __ str(src->as_register(), as_Address(to_addr));
 869       break;
 870     case T_ADDRESS:
 871       __ str(src->as_register(), as_Address(to_addr));
 872       break;
 873     case T_INT:
 874       __ strw(src->as_register(), as_Address(to_addr));
 875       break;
 876 
 877     case T_LONG: {
 878       __ str(src->as_register_lo(), as_Address_lo(to_addr));
 879       break;
 880     }
 881 
 882     case T_BYTE:    // fall through
 883     case T_BOOLEAN: {
 884       __ strb(src->as_register(), as_Address(to_addr));
 885       break;
 886     }
 887 
 888     case T_CHAR:    // fall through
 889     case T_SHORT:
 890       __ strh(src->as_register(), as_Address(to_addr));
 891       break;
 892 
 893     default:
 894       ShouldNotReachHere();
 895   }
 896   if (info != NULL) {
 897     add_debug_info_for_null_check(null_check_here, info);
 898   }
 899 }
 900 
 901 
 902 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 903   assert(src->is_stack(), "should not call otherwise");
 904   assert(dest->is_register(), "should not call otherwise");
 905 
 906   if (dest->is_single_cpu()) {
 907     if (type == T_ARRAY || type == T_OBJECT) {
 908       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 909       __ verify_oop(dest->as_register());
 910     } else if (type == T_METADATA || type == T_ADDRESS) {
 911       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 912     } else {
 913       __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 914     }
 915 
 916   } else if (dest->is_double_cpu()) {
 917     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 918     __ ldr(dest->as_register_lo(), src_addr_LO);
 919 
 920   } else if (dest->is_single_fpu()) {
 921     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 922     __ ldrs(dest->as_float_reg(), src_addr);
 923 
 924   } else if (dest->is_double_fpu()) {
 925     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 926     __ ldrd(dest->as_double_reg(), src_addr);
 927 
 928   } else {
 929     ShouldNotReachHere();
 930   }
 931 }
 932 
 933 
 934 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 935   address target = NULL;
 936   relocInfo::relocType reloc_type = relocInfo::none;
 937 
 938   switch (patching_id(info)) {
 939   case PatchingStub::access_field_id:
 940     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 941     reloc_type = relocInfo::section_word_type;
 942     break;
 943   case PatchingStub::load_klass_id:
 944     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 945     reloc_type = relocInfo::metadata_type;
 946     break;
 947   case PatchingStub::load_mirror_id:
 948     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 949     reloc_type = relocInfo::oop_type;
 950     break;
 951   case PatchingStub::load_appendix_id:
 952     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 953     reloc_type = relocInfo::oop_type;
 954     break;
 955   default: ShouldNotReachHere();
 956   }
 957 
 958   __ far_call(RuntimeAddress(target));
 959   add_call_info_here(info);
 960 }
 961 
 962 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 963 
 964   LIR_Opr temp;
 965   if (type == T_LONG || type == T_DOUBLE)
 966     temp = FrameMap::rscratch1_long_opr;
 967   else
 968     temp = FrameMap::rscratch1_opr;
 969 
 970   stack2reg(src, temp, src->type());
 971   reg2stack(temp, dest, dest->type(), false);
 972 }
 973 
 974 
 975 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
 976   LIR_Address* addr = src->as_address_ptr();
 977   LIR_Address* from_addr = src->as_address_ptr();
 978 
 979   if (addr->base()->type() == T_OBJECT) {
 980     __ verify_oop(addr->base()->as_pointer_register());
 981   }
 982 
 983   if (patch_code != lir_patch_none) {
 984     deoptimize_trap(info);
 985     return;
 986   }
 987 
 988   if (info != NULL) {
 989     add_debug_info_for_null_check_here(info);
 990   }
 991   int null_check_here = code_offset();
 992   switch (type) {
 993     case T_FLOAT: {
 994       __ ldrs(dest->as_float_reg(), as_Address(from_addr));
 995       break;
 996     }
 997 
 998     case T_DOUBLE: {
 999       __ ldrd(dest->as_double_reg(), as_Address(from_addr));
1000       break;
1001     }
1002 
1003     case T_ARRAY:   // fall through
1004     case T_OBJECT:  // fall through
1005       if (UseCompressedOops && !wide) {
1006         __ ldrw(dest->as_register(), as_Address(from_addr));
1007       } else {
1008          __ ldr(dest->as_register(), as_Address(from_addr));
1009       }
1010       break;
1011     case T_METADATA:
1012       // We get here to store a method pointer to the stack to pass to
1013       // a dtrace runtime call. This can't work on 64 bit with
1014       // compressed klass ptrs: T_METADATA can be a compressed klass
1015       // ptr or a 64 bit method pointer.
1016       LP64_ONLY(ShouldNotReachHere());
1017       __ ldr(dest->as_register(), as_Address(from_addr));
1018       break;
1019     case T_ADDRESS:
1020       // FIXME: OMG this is a horrible kludge.  Any offset from an
1021       // address that matches klass_offset_in_bytes() will be loaded
1022       // as a word, not a long.
1023       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1024         __ ldrw(dest->as_register(), as_Address(from_addr));
1025       } else {
1026         __ ldr(dest->as_register(), as_Address(from_addr));
1027       }
1028       break;
1029     case T_INT:
1030       __ ldrw(dest->as_register(), as_Address(from_addr));
1031       break;
1032 
1033     case T_LONG: {
1034       __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
1035       break;
1036     }
1037 
1038     case T_BYTE:
1039       __ ldrsb(dest->as_register(), as_Address(from_addr));
1040       break;
1041     case T_BOOLEAN: {
1042       __ ldrb(dest->as_register(), as_Address(from_addr));
1043       break;
1044     }
1045 
1046     case T_CHAR:
1047       __ ldrh(dest->as_register(), as_Address(from_addr));
1048       break;
1049     case T_SHORT:
1050       __ ldrsh(dest->as_register(), as_Address(from_addr));
1051       break;
1052 
1053     default:
1054       ShouldNotReachHere();
1055   }
1056 
1057   if (type == T_ARRAY || type == T_OBJECT) {
1058 #ifdef _LP64
1059     if (UseCompressedOops && !wide) {
1060       __ decode_heap_oop(dest->as_register());
1061     }
1062 #endif
1063     __ verify_oop(dest->as_register());
1064   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1065 #ifdef _LP64
1066     if (UseCompressedClassPointers) {
1067       __ decode_klass_not_null(dest->as_register());
1068     }
1069 #endif
1070   }
1071 }
1072 
1073 
1074 void LIR_Assembler::prefetchr(LIR_Opr src) { Unimplemented(); }
1075 
1076 
1077 void LIR_Assembler::prefetchw(LIR_Opr src) { Unimplemented(); }
1078 
1079 
1080 int LIR_Assembler::array_element_size(BasicType type) const {
1081   int elem_size = type2aelembytes(type);
1082   return exact_log2(elem_size);
1083 }
1084 
1085 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1086   Register Rdividend = op->in_opr1()->as_register();
1087   Register Rdivisor  = op->in_opr2()->as_register();
1088   Register Rscratch  = op->in_opr3()->as_register();
1089   Register Rresult   = op->result_opr()->as_register();
1090   int divisor = -1;
1091 
1092   /*
1093   TODO: For some reason, using the Rscratch that gets passed in is
1094   not possible because the register allocator does not see the tmp reg
1095   as used, and assignes it the same register as Rdividend. We use rscratch1
1096    instead.
1097 
1098   assert(Rdividend != Rscratch, "");
1099   assert(Rdivisor  != Rscratch, "");
1100   */
1101 
1102   if (Rdivisor == noreg && is_power_of_2(divisor)) {
1103     // convert division by a power of two into some shifts and logical operations
1104   }
1105 
1106   if (op->code() == lir_irem) {
1107     __ corrected_idivl(Rresult, Rdividend, Rdivisor, true, rscratch1);
1108    } else if (op->code() == lir_idiv) {
1109     __ corrected_idivl(Rresult, Rdividend, Rdivisor, false, rscratch1);
1110   } else
1111     ShouldNotReachHere();
1112 }
1113 
1114 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1115 #ifdef ASSERT
1116   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1117   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1118   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1119 #endif
1120 
1121   if (op->cond() == lir_cond_always) {
1122     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1123     __ b(*(op->label()));
1124   } else {
1125     Assembler::Condition acond;
1126     if (op->code() == lir_cond_float_branch) {
1127       bool is_unordered = (op->ublock() == op->block());
1128       // Assembler::EQ does not permit unordered branches, so we add
1129       // another branch here.  Likewise, Assembler::NE does not permit
1130       // ordered branches.
1131       if ((is_unordered && op->cond() == lir_cond_equal)
1132           || (!is_unordered && op->cond() == lir_cond_notEqual))
1133         __ br(Assembler::VS, *(op->ublock()->label()));
1134       switch(op->cond()) {
1135       case lir_cond_equal:        acond = Assembler::EQ; break;
1136       case lir_cond_notEqual:     acond = Assembler::NE; break;
1137       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1138       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1139       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1140       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1141       default:                    ShouldNotReachHere();
1142       }
1143     } else {
1144       switch (op->cond()) {
1145         case lir_cond_equal:        acond = Assembler::EQ; break;
1146         case lir_cond_notEqual:     acond = Assembler::NE; break;
1147         case lir_cond_less:         acond = Assembler::LT; break;
1148         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1149         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1150         case lir_cond_greater:      acond = Assembler::GT; break;
1151         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1152         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1153         default:                         ShouldNotReachHere();
1154       }
1155     }
1156     __ br(acond,*(op->label()));
1157   }
1158 }
1159 
1160 
1161 
1162 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1163   LIR_Opr src  = op->in_opr();
1164   LIR_Opr dest = op->result_opr();
1165 
1166   switch (op->bytecode()) {
1167     case Bytecodes::_i2f:
1168       {
1169         __ scvtfws(dest->as_float_reg(), src->as_register());
1170         break;
1171       }
1172     case Bytecodes::_i2d:
1173       {
1174         __ scvtfwd(dest->as_double_reg(), src->as_register());
1175         break;
1176       }
1177     case Bytecodes::_l2d:
1178       {
1179         __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1180         break;
1181       }
1182     case Bytecodes::_l2f:
1183       {
1184         __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1185         break;
1186       }
1187     case Bytecodes::_f2d:
1188       {
1189         __ fcvts(dest->as_double_reg(), src->as_float_reg());
1190         break;
1191       }
1192     case Bytecodes::_d2f:
1193       {
1194         __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1195         break;
1196       }
1197     case Bytecodes::_i2c:
1198       {
1199         __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1200         break;
1201       }
1202     case Bytecodes::_i2l:
1203       {
1204         __ sxtw(dest->as_register_lo(), src->as_register());
1205         break;
1206       }
1207     case Bytecodes::_i2s:
1208       {
1209         __ sxth(dest->as_register(), src->as_register());
1210         break;
1211       }
1212     case Bytecodes::_i2b:
1213       {
1214         __ sxtb(dest->as_register(), src->as_register());
1215         break;
1216       }
1217     case Bytecodes::_l2i:
1218       {
1219         _masm->block_comment("FIXME: This could be a no-op");
1220         __ uxtw(dest->as_register(), src->as_register_lo());
1221         break;
1222       }
1223     case Bytecodes::_d2l:
1224       {
1225         Register tmp = op->tmp1()->as_register();
1226         __ clear_fpsr();
1227         __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1228         __ get_fpsr(tmp);
1229         __ tst(tmp, 1); // FPSCR.IOC
1230         __ br(Assembler::NE, *(op->stub()->entry()));
1231         __ bind(*op->stub()->continuation());
1232         break;
1233       }
1234     case Bytecodes::_f2i:
1235       {
1236         Register tmp = op->tmp1()->as_register();
1237         __ clear_fpsr();
1238         __ fcvtzsw(dest->as_register(), src->as_float_reg());
1239         __ get_fpsr(tmp);
1240         __ tst(tmp, 1); // FPSCR.IOC
1241         __ br(Assembler::NE, *(op->stub()->entry()));
1242         __ bind(*op->stub()->continuation());
1243         break;
1244       }
1245     case Bytecodes::_f2l:
1246       {
1247         Register tmp = op->tmp1()->as_register();
1248         __ clear_fpsr();
1249         __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1250         __ get_fpsr(tmp);
1251         __ tst(tmp, 1); // FPSCR.IOC
1252         __ br(Assembler::NE, *(op->stub()->entry()));
1253         __ bind(*op->stub()->continuation());
1254         break;
1255       }
1256     case Bytecodes::_d2i:
1257       {
1258         Register tmp = op->tmp1()->as_register();
1259         __ clear_fpsr();
1260         __ fcvtzdw(dest->as_register(), src->as_double_reg());
1261         __ get_fpsr(tmp);
1262         __ tst(tmp, 1); // FPSCR.IOC
1263         __ br(Assembler::NE, *(op->stub()->entry()));
1264         __ bind(*op->stub()->continuation());
1265         break;
1266       }
1267     default: ShouldNotReachHere();
1268   }
1269 }
1270 
1271 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1272   if (op->init_check()) {
1273     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1274                                InstanceKlass::init_state_offset()));
1275     __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1276     add_debug_info_for_null_check_here(op->stub()->info());
1277     __ br(Assembler::NE, *op->stub()->entry());
1278   }
1279   __ allocate_object(op->obj()->as_register(),
1280                      op->tmp1()->as_register(),
1281                      op->tmp2()->as_register(),
1282                      op->header_size(),
1283                      op->object_size(),
1284                      op->klass()->as_register(),
1285                      *op->stub()->entry());
1286   __ bind(*op->stub()->continuation());
1287 }
1288 
1289 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1290   Register len =  op->len()->as_register();
1291   __ uxtw(len, len);
1292 
1293   if (UseSlowPath ||
1294       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1295       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1296     __ b(*op->stub()->entry());
1297   } else {
1298     Register tmp1 = op->tmp1()->as_register();
1299     Register tmp2 = op->tmp2()->as_register();
1300     Register tmp3 = op->tmp3()->as_register();
1301     if (len == tmp1) {
1302       tmp1 = tmp3;
1303     } else if (len == tmp2) {
1304       tmp2 = tmp3;
1305     } else if (len == tmp3) {
1306       // everything is ok
1307     } else {
1308       __ mov(tmp3, len);
1309     }
1310     __ allocate_array(op->obj()->as_register(),
1311                       len,
1312                       tmp1,
1313                       tmp2,
1314                       arrayOopDesc::header_size(op->type()),
1315                       array_element_size(op->type()),
1316                       op->klass()->as_register(),
1317                       *op->stub()->entry());
1318   }
1319   __ bind(*op->stub()->continuation());
1320 }
1321 
1322 void LIR_Assembler::type_profile_helper(Register mdo,
1323                                         ciMethodData *md, ciProfileData *data,
1324                                         Register recv, Label* update_done) {
1325   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1326     Label next_test;
1327     // See if the receiver is receiver[n].
1328     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1329     __ ldr(rscratch1, Address(rscratch2));
1330     __ cmp(recv, rscratch1);
1331     __ br(Assembler::NE, next_test);
1332     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1333     __ addptr(data_addr, DataLayout::counter_increment);
1334     __ b(*update_done);
1335     __ bind(next_test);
1336   }
1337 
1338   // Didn't find receiver; find next empty slot and fill it in
1339   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1340     Label next_test;
1341     __ lea(rscratch2,
1342            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1343     Address recv_addr(rscratch2);
1344     __ ldr(rscratch1, recv_addr);
1345     __ cbnz(rscratch1, next_test);
1346     __ str(recv, recv_addr);
1347     __ mov(rscratch1, DataLayout::counter_increment);
1348     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1349     __ str(rscratch1, Address(rscratch2));
1350     __ b(*update_done);
1351     __ bind(next_test);
1352   }
1353 }
1354 
1355 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1356   // we always need a stub for the failure case.
1357   CodeStub* stub = op->stub();
1358   Register obj = op->object()->as_register();
1359   Register k_RInfo = op->tmp1()->as_register();
1360   Register klass_RInfo = op->tmp2()->as_register();
1361   Register dst = op->result_opr()->as_register();
1362   ciKlass* k = op->klass();
1363   Register Rtmp1 = noreg;
1364 
1365   // check if it needs to be profiled
1366   ciMethodData* md;
1367   ciProfileData* data;
1368 
1369   if (op->should_profile()) {
1370     ciMethod* method = op->profiled_method();
1371     assert(method != NULL, "Should have method");
1372     int bci = op->profiled_bci();
1373     md = method->method_data_or_null();
1374     assert(md != NULL, "Sanity");
1375     data = md->bci_to_data(bci);
1376     assert(data != NULL,                "need data for type check");
1377     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1378   }
1379   Label profile_cast_success, profile_cast_failure;
1380   Label *success_target = op->should_profile() ? &profile_cast_success : success;
1381   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1382 
1383   if (obj == k_RInfo) {
1384     k_RInfo = dst;
1385   } else if (obj == klass_RInfo) {
1386     klass_RInfo = dst;
1387   }
1388   if (k->is_loaded() && !UseCompressedClassPointers) {
1389     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1390   } else {
1391     Rtmp1 = op->tmp3()->as_register();
1392     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1393   }
1394 
1395   assert_different_registers(obj, k_RInfo, klass_RInfo);
1396 
1397     if (op->should_profile()) {
1398       Label not_null;
1399       __ cbnz(obj, not_null);
1400       // Object is null; update MDO and exit
1401       Register mdo  = klass_RInfo;
1402       __ mov_metadata(mdo, md->constant_encoding());
1403       Address data_addr
1404         = __ form_address(rscratch2, mdo,
1405                           md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1406                           LogBytesPerWord);
1407       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1408       __ ldr(rscratch1, data_addr);
1409       __ orr(rscratch1, rscratch1, header_bits);
1410       __ str(rscratch1, data_addr);
1411       __ b(*obj_is_null);
1412       __ bind(not_null);
1413     } else {
1414       __ cbz(obj, *obj_is_null);
1415     }
1416 
1417   if (!k->is_loaded()) {
1418     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1419   } else {
1420 #ifdef _LP64
1421     __ mov_metadata(k_RInfo, k->constant_encoding());
1422 #endif // _LP64
1423   }
1424   __ verify_oop(obj);
1425 
1426   if (op->fast_check()) {
1427     // get object class
1428     // not a safepoint as obj null check happens earlier
1429     __ load_klass(rscratch1, obj);
1430     __ cmp( rscratch1, k_RInfo);
1431 
1432     __ br(Assembler::NE, *failure_target);
1433     // successful cast, fall through to profile or jump
1434   } else {
1435     // get object class
1436     // not a safepoint as obj null check happens earlier
1437     __ load_klass(klass_RInfo, obj);
1438     if (k->is_loaded()) {
1439       // See if we get an immediate positive hit
1440       __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1441       __ cmp(k_RInfo, rscratch1);
1442       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1443         __ br(Assembler::NE, *failure_target);
1444         // successful cast, fall through to profile or jump
1445       } else {
1446         // See if we get an immediate positive hit
1447         __ br(Assembler::EQ, *success_target);
1448         // check for self
1449         __ cmp(klass_RInfo, k_RInfo);
1450         __ br(Assembler::EQ, *success_target);
1451 
1452         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1453         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1454         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1455         // result is a boolean
1456         __ cbzw(klass_RInfo, *failure_target);
1457         // successful cast, fall through to profile or jump
1458       }
1459     } else {
1460       // perform the fast part of the checking logic
1461       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1462       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1463       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1464       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1465       __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1466       // result is a boolean
1467       __ cbz(k_RInfo, *failure_target);
1468       // successful cast, fall through to profile or jump
1469     }
1470   }
1471   if (op->should_profile()) {
1472     Register mdo  = klass_RInfo, recv = k_RInfo;
1473     __ bind(profile_cast_success);
1474     __ mov_metadata(mdo, md->constant_encoding());
1475     __ load_klass(recv, obj);
1476     Label update_done;
1477     type_profile_helper(mdo, md, data, recv, success);
1478     __ b(*success);
1479 
1480     __ bind(profile_cast_failure);
1481     __ mov_metadata(mdo, md->constant_encoding());
1482     Address counter_addr
1483       = __ form_address(rscratch2, mdo,
1484                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1485                         LogBytesPerWord);
1486     __ ldr(rscratch1, counter_addr);
1487     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1488     __ str(rscratch1, counter_addr);
1489     __ b(*failure);
1490   }
1491   __ b(*success);
1492 }
1493 
1494 
1495 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1496   LIR_Code code = op->code();
1497   if (code == lir_store_check) {
1498     Register value = op->object()->as_register();
1499     Register array = op->array()->as_register();
1500     Register k_RInfo = op->tmp1()->as_register();
1501     Register klass_RInfo = op->tmp2()->as_register();
1502     Register Rtmp1 = op->tmp3()->as_register();
1503 
1504     CodeStub* stub = op->stub();
1505 
1506     // check if it needs to be profiled
1507     ciMethodData* md;
1508     ciProfileData* data;
1509 
1510     if (op->should_profile()) {
1511       ciMethod* method = op->profiled_method();
1512       assert(method != NULL, "Should have method");
1513       int bci = op->profiled_bci();
1514       md = method->method_data_or_null();
1515       assert(md != NULL, "Sanity");
1516       data = md->bci_to_data(bci);
1517       assert(data != NULL,                "need data for type check");
1518       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1519     }
1520     Label profile_cast_success, profile_cast_failure, done;
1521     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1522     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1523 
1524     if (op->should_profile()) {
1525       Label not_null;
1526       __ cbnz(value, not_null);
1527       // Object is null; update MDO and exit
1528       Register mdo  = klass_RInfo;
1529       __ mov_metadata(mdo, md->constant_encoding());
1530       Address data_addr
1531         = __ form_address(rscratch2, mdo,
1532                           md->byte_offset_of_slot(data, DataLayout::header_offset()),
1533                           LogBytesPerInt);
1534       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1535       __ ldrw(rscratch1, data_addr);
1536       __ orrw(rscratch1, rscratch1, header_bits);
1537       __ strw(rscratch1, data_addr);
1538       __ b(done);
1539       __ bind(not_null);
1540     } else {
1541       __ cbz(value, done);
1542     }
1543 
1544     add_debug_info_for_null_check_here(op->info_for_exception());
1545     __ load_klass(k_RInfo, array);
1546     __ load_klass(klass_RInfo, value);
1547 
1548     // get instance klass (it's already uncompressed)
1549     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1550     // perform the fast part of the checking logic
1551     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1552     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1553     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1554     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1555     __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1556     // result is a boolean
1557     __ cbzw(k_RInfo, *failure_target);
1558     // fall through to the success case
1559 
1560     if (op->should_profile()) {
1561       Register mdo  = klass_RInfo, recv = k_RInfo;
1562       __ bind(profile_cast_success);
1563       __ mov_metadata(mdo, md->constant_encoding());
1564       __ load_klass(recv, value);
1565       Label update_done;
1566       type_profile_helper(mdo, md, data, recv, &done);
1567       __ b(done);
1568 
1569       __ bind(profile_cast_failure);
1570       __ mov_metadata(mdo, md->constant_encoding());
1571       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1572       __ lea(rscratch2, counter_addr);
1573       __ ldr(rscratch1, Address(rscratch2));
1574       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1575       __ str(rscratch1, Address(rscratch2));
1576       __ b(*stub->entry());
1577     }
1578 
1579     __ bind(done);
1580   } else if (code == lir_checkcast) {
1581     Register obj = op->object()->as_register();
1582     Register dst = op->result_opr()->as_register();
1583     Label success;
1584     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1585     __ bind(success);
1586     if (dst != obj) {
1587       __ mov(dst, obj);
1588     }
1589   } else if (code == lir_instanceof) {
1590     Register obj = op->object()->as_register();
1591     Register dst = op->result_opr()->as_register();
1592     Label success, failure, done;
1593     emit_typecheck_helper(op, &success, &failure, &failure);
1594     __ bind(failure);
1595     __ mov(dst, zr);
1596     __ b(done);
1597     __ bind(success);
1598     __ mov(dst, 1);
1599     __ bind(done);
1600   } else {
1601     ShouldNotReachHere();
1602   }
1603 }
1604 
1605 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1606   __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, rscratch1);
1607   __ cset(rscratch1, Assembler::NE);
1608   __ membar(__ AnyAny);
1609 }
1610 
1611 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1612   __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, rscratch1);
1613   __ cset(rscratch1, Assembler::NE);
1614   __ membar(__ AnyAny);
1615 }
1616 
1617 
1618 // Return 1 in rscratch1 if the CAS fails.
1619 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1620   assert(VM_Version::supports_cx8(), "wrong machine");
1621   Register addr = as_reg(op->addr());
1622   Register newval = as_reg(op->new_value());
1623   Register cmpval = as_reg(op->cmp_value());
1624   Label succeed, fail, around;
1625   Register res = op->result_opr()->as_register();
1626 
1627   if (op->code() == lir_cas_obj) {
1628     assert(op->tmp1()->is_valid(), "must be");
1629     Register t1 = op->tmp1()->as_register();
1630     if (UseCompressedOops) {
1631 #if INCLUDE_ALL_GCS
1632       if (UseShenandoahGC && ShenandoahCASBarrier) {
1633         __ encode_heap_oop(t1, cmpval);
1634         cmpval = t1;
1635         assert(op->tmp2()->is_valid(), "must be");
1636         Register t2 = op->tmp2()->as_register();
1637         __ encode_heap_oop(t2, newval);
1638         newval = t2;
1639         ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(_masm, addr, cmpval, newval, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, res);
1640       } else
1641 #endif
1642       {
1643         __ encode_heap_oop(t1, cmpval);
1644         cmpval = t1;
1645         __ encode_heap_oop(rscratch2, newval);
1646         newval = rscratch2;
1647         casw(addr, newval, cmpval);
1648         __ eorw (res, r8, 1);
1649       }
1650     } else {
1651 #if INCLUDE_ALL_GCS
1652       if (UseShenandoahGC && ShenandoahCASBarrier) {
1653         ShenandoahBarrierSetAssembler::bsasm()->cmpxchg_oop(_masm, addr, cmpval, newval, /*acquire*/ false, /*release*/ true, /*weak*/ false, /*is_cae*/ false, res);
1654       } else
1655 #endif
1656       {
1657         casl(addr, newval, cmpval);
1658         __ eorw (res, r8, 1);
1659       }
1660     }
1661   } else if (op->code() == lir_cas_int) {
1662     casw(addr, newval, cmpval);
1663     __ eorw (res, r8, 1);
1664   } else {
1665     casl(addr, newval, cmpval);
1666     __ eorw (res, r8, 1);
1667   }
1668 }
1669 
1670 
1671 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1672 
1673   Assembler::Condition acond, ncond;
1674   switch (condition) {
1675   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1676   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1677   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1678   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1679   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1680   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1681   case lir_cond_belowEqual:   Unimplemented(); break;
1682   case lir_cond_aboveEqual:   Unimplemented(); break;
1683   default:                    ShouldNotReachHere();
1684   }
1685 
1686   assert(result->is_single_cpu() || result->is_double_cpu(),
1687          "expect single register for result");
1688   if (opr1->is_constant() && opr2->is_constant()
1689       && opr1->type() == T_INT && opr2->type() == T_INT) {
1690     jint val1 = opr1->as_jint();
1691     jint val2 = opr2->as_jint();
1692     if (val1 == 0 && val2 == 1) {
1693       __ cset(result->as_register(), ncond);
1694       return;
1695     } else if (val1 == 1 && val2 == 0) {
1696       __ cset(result->as_register(), acond);
1697       return;
1698     }
1699   }
1700 
1701   if (opr1->is_constant() && opr2->is_constant()
1702       && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1703     jlong val1 = opr1->as_jlong();
1704     jlong val2 = opr2->as_jlong();
1705     if (val1 == 0 && val2 == 1) {
1706       __ cset(result->as_register_lo(), ncond);
1707       return;
1708     } else if (val1 == 1 && val2 == 0) {
1709       __ cset(result->as_register_lo(), acond);
1710       return;
1711     }
1712   }
1713 
1714   if (opr1->is_stack()) {
1715     stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1716     opr1 = FrameMap::rscratch1_opr;
1717   } else if (opr1->is_constant()) {
1718     LIR_Opr tmp
1719       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1720     const2reg(opr1, tmp, lir_patch_none, NULL);
1721     opr1 = tmp;
1722   }
1723 
1724   if (opr2->is_stack()) {
1725     stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1726     opr2 = FrameMap::rscratch2_opr;
1727   } else if (opr2->is_constant()) {
1728     LIR_Opr tmp
1729       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1730     const2reg(opr2, tmp, lir_patch_none, NULL);
1731     opr2 = tmp;
1732   }
1733 
1734   if (result->type() == T_LONG)
1735     __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1736   else
1737     __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1738 }
1739 
1740 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1741   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1742 
1743   if (left->is_single_cpu()) {
1744     Register lreg = left->as_register();
1745     Register dreg = as_reg(dest);
1746 
1747     if (right->is_single_cpu()) {
1748       // cpu register - cpu register
1749 
1750       assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1751              "should be");
1752       Register rreg = right->as_register();
1753       switch (code) {
1754       case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1755       case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1756       case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1757       default:      ShouldNotReachHere();
1758       }
1759 
1760     } else if (right->is_double_cpu()) {
1761       Register rreg = right->as_register_lo();
1762       // single_cpu + double_cpu: can happen with obj+long
1763       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1764       switch (code) {
1765       case lir_add: __ add(dreg, lreg, rreg); break;
1766       case lir_sub: __ sub(dreg, lreg, rreg); break;
1767       default: ShouldNotReachHere();
1768       }
1769     } else if (right->is_constant()) {
1770       // cpu register - constant
1771       jlong c;
1772 
1773       // FIXME.  This is fugly: we really need to factor all this logic.
1774       switch(right->type()) {
1775       case T_LONG:
1776         c = right->as_constant_ptr()->as_jlong();
1777         break;
1778       case T_INT:
1779       case T_ADDRESS:
1780         c = right->as_constant_ptr()->as_jint();
1781         break;
1782       default:
1783         ShouldNotReachHere();
1784         break;
1785       }
1786 
1787       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1788       if (c == 0 && dreg == lreg) {
1789         COMMENT("effective nop elided");
1790         return;
1791       }
1792       switch(left->type()) {
1793       case T_INT:
1794         switch (code) {
1795         case lir_add: __ addw(dreg, lreg, c); break;
1796         case lir_sub: __ subw(dreg, lreg, c); break;
1797         default: ShouldNotReachHere();
1798         }
1799         break;
1800       case T_OBJECT:
1801       case T_ADDRESS:
1802         switch (code) {
1803         case lir_add: __ add(dreg, lreg, c); break;
1804         case lir_sub: __ sub(dreg, lreg, c); break;
1805         default: ShouldNotReachHere();
1806         }
1807         break;
1808         ShouldNotReachHere();
1809       }
1810     } else {
1811       ShouldNotReachHere();
1812     }
1813 
1814   } else if (left->is_double_cpu()) {
1815     Register lreg_lo = left->as_register_lo();
1816 
1817     if (right->is_double_cpu()) {
1818       // cpu register - cpu register
1819       Register rreg_lo = right->as_register_lo();
1820       switch (code) {
1821       case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1822       case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1823       case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1824       case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1825       case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1826       default:
1827         ShouldNotReachHere();
1828       }
1829 
1830     } else if (right->is_constant()) {
1831       jlong c = right->as_constant_ptr()->as_jlong_bits();
1832       Register dreg = as_reg(dest);
1833       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1834       if (c == 0 && dreg == lreg_lo) {
1835         COMMENT("effective nop elided");
1836         return;
1837       }
1838       switch (code) {
1839         case lir_add: __ add(dreg, lreg_lo, c); break;
1840         case lir_sub: __ sub(dreg, lreg_lo, c); break;
1841         default:
1842           ShouldNotReachHere();
1843       }
1844     } else {
1845       ShouldNotReachHere();
1846     }
1847   } else if (left->is_single_fpu()) {
1848     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1849     switch (code) {
1850     case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1851     case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1852     case lir_mul_strictfp: // fall through
1853     case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1854     case lir_div_strictfp: // fall through
1855     case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1856     default:
1857       ShouldNotReachHere();
1858     }
1859   } else if (left->is_double_fpu()) {
1860     if (right->is_double_fpu()) {
1861       // fpu register - fpu register
1862       switch (code) {
1863       case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1864       case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1865       case lir_mul_strictfp: // fall through
1866       case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1867       case lir_div_strictfp: // fall through
1868       case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1869       default:
1870         ShouldNotReachHere();
1871       }
1872     } else {
1873       if (right->is_constant()) {
1874         ShouldNotReachHere();
1875       }
1876       ShouldNotReachHere();
1877     }
1878   } else if (left->is_single_stack() || left->is_address()) {
1879     assert(left == dest, "left and dest must be equal");
1880     ShouldNotReachHere();
1881   } else {
1882     ShouldNotReachHere();
1883   }
1884 }
1885 
1886 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1887 
1888 
1889 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1890   switch(code) {
1891   case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1892   case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1893   default      : ShouldNotReachHere();
1894   }
1895 }
1896 
1897 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1898 
1899   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1900   Register Rleft = left->is_single_cpu() ? left->as_register() :
1901                                            left->as_register_lo();
1902    if (dst->is_single_cpu()) {
1903      Register Rdst = dst->as_register();
1904      if (right->is_constant()) {
1905        switch (code) {
1906          case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1907          case lir_logic_or:  __ orrw (Rdst, Rleft, right->as_jint()); break;
1908          case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1909          default: ShouldNotReachHere(); break;
1910        }
1911      } else {
1912        Register Rright = right->is_single_cpu() ? right->as_register() :
1913                                                   right->as_register_lo();
1914        switch (code) {
1915          case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1916          case lir_logic_or:  __ orrw (Rdst, Rleft, Rright); break;
1917          case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1918          default: ShouldNotReachHere(); break;
1919        }
1920      }
1921    } else {
1922      Register Rdst = dst->as_register_lo();
1923      if (right->is_constant()) {
1924        switch (code) {
1925          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1926          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jlong()); break;
1927          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1928          default: ShouldNotReachHere(); break;
1929        }
1930      } else {
1931        Register Rright = right->is_single_cpu() ? right->as_register() :
1932                                                   right->as_register_lo();
1933        switch (code) {
1934          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1935          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
1936          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1937          default: ShouldNotReachHere(); break;
1938        }
1939      }
1940    }
1941 }
1942 
1943 
1944 
1945 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1946 
1947 
1948 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1949   if (opr1->is_constant() && opr2->is_single_cpu()) {
1950     // tableswitch
1951     Register reg = as_reg(opr2);
1952     struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1953     __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1954   } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1955     Register reg1 = as_reg(opr1);
1956     if (opr2->is_single_cpu()) {
1957       // cpu register - cpu register
1958       Register reg2 = opr2->as_register();
1959       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1960         __ cmp(reg1, reg2);
1961       } else {
1962         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1963         __ cmpw(reg1, reg2);
1964       }
1965       return;
1966     }
1967     if (opr2->is_double_cpu()) {
1968       // cpu register - cpu register
1969       Register reg2 = opr2->as_register_lo();
1970       __ cmp(reg1, reg2);
1971       return;
1972     }
1973 
1974     if (opr2->is_constant()) {
1975       bool is_32bit = false; // width of register operand
1976       jlong imm;
1977 
1978       switch(opr2->type()) {
1979       case T_INT:
1980         imm = opr2->as_constant_ptr()->as_jint();
1981         is_32bit = true;
1982         break;
1983       case T_LONG:
1984         imm = opr2->as_constant_ptr()->as_jlong();
1985         break;
1986       case T_ADDRESS:
1987         imm = opr2->as_constant_ptr()->as_jint();
1988         break;
1989       case T_OBJECT:
1990       case T_ARRAY:
1991         imm = jlong(opr2->as_constant_ptr()->as_jobject());
1992         break;
1993       default:
1994         ShouldNotReachHere();
1995         break;
1996       }
1997 
1998       if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1999         if (is_32bit)
2000           __ cmpw(reg1, imm);
2001         else
2002           __ cmp(reg1, imm);
2003         return;
2004       } else {
2005         __ mov(rscratch1, imm);
2006         if (is_32bit)
2007           __ cmpw(reg1, rscratch1);
2008         else
2009           __ cmp(reg1, rscratch1);
2010         return;
2011       }
2012     } else
2013       ShouldNotReachHere();
2014   } else if (opr1->is_single_fpu()) {
2015     FloatRegister reg1 = opr1->as_float_reg();
2016     assert(opr2->is_single_fpu(), "expect single float register");
2017     FloatRegister reg2 = opr2->as_float_reg();
2018     __ fcmps(reg1, reg2);
2019   } else if (opr1->is_double_fpu()) {
2020     FloatRegister reg1 = opr1->as_double_reg();
2021     assert(opr2->is_double_fpu(), "expect double float register");
2022     FloatRegister reg2 = opr2->as_double_reg();
2023     __ fcmpd(reg1, reg2);
2024   } else {
2025     ShouldNotReachHere();
2026   }
2027 }
2028 
2029 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
2030   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
2031     bool is_unordered_less = (code == lir_ucmp_fd2i);
2032     if (left->is_single_fpu()) {
2033       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
2034     } else if (left->is_double_fpu()) {
2035       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
2036     } else {
2037       ShouldNotReachHere();
2038     }
2039   } else if (code == lir_cmp_l2i) {
2040     Label done;
2041     __ cmp(left->as_register_lo(), right->as_register_lo());
2042     __ mov(dst->as_register(), (u_int64_t)-1L);
2043     __ br(Assembler::LT, done);
2044     __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
2045     __ bind(done);
2046   } else {
2047     ShouldNotReachHere();
2048   }
2049 }
2050 
2051 
2052 void LIR_Assembler::align_call(LIR_Code code) {  }
2053 
2054 
2055 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2056   address call = __ trampoline_call(Address(op->addr(), rtype));
2057   if (call == NULL) {
2058     bailout("trampoline stub overflow");
2059     return;
2060   }
2061   add_call_info(code_offset(), op->info());
2062 }
2063 
2064 
2065 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2066   address call = __ ic_call(op->addr());
2067   if (call == NULL) {
2068     bailout("trampoline stub overflow");
2069     return;
2070   }
2071   add_call_info(code_offset(), op->info());
2072 }
2073 
2074 
2075 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2076 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2077   ShouldNotReachHere();
2078 }
2079 
2080 
2081 void LIR_Assembler::emit_static_call_stub() {
2082   address call_pc = __ pc();
2083   address stub = __ start_a_stub(call_stub_size);
2084   if (stub == NULL) {
2085     bailout("static call stub overflow");
2086     return;
2087   }
2088 
2089   int start = __ offset();
2090 
2091   __ relocate(static_stub_Relocation::spec(call_pc));
2092   __ mov_metadata(rmethod, (Metadata*)NULL);
2093   __ movptr(rscratch1, 0);
2094   __ br(rscratch1);
2095 
2096   assert(__ offset() - start <= call_stub_size, "stub too big");
2097   __ end_a_stub();
2098 }
2099 
2100 
2101 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2102   assert(exceptionOop->as_register() == r0, "must match");
2103   assert(exceptionPC->as_register() == r3, "must match");
2104 
2105   // exception object is not added to oop map by LinearScan
2106   // (LinearScan assumes that no oops are in fixed registers)
2107   info->add_register_oop(exceptionOop);
2108   Runtime1::StubID unwind_id;
2109 
2110   // get current pc information
2111   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2112   int pc_for_athrow_offset = __ offset();
2113   InternalAddress pc_for_athrow(__ pc());
2114   __ adr(exceptionPC->as_register(), pc_for_athrow);
2115   add_call_info(pc_for_athrow_offset, info); // for exception handler
2116 
2117   __ verify_not_null_oop(r0);
2118   // search an exception handler (r0: exception oop, r3: throwing pc)
2119   if (compilation()->has_fpu_code()) {
2120     unwind_id = Runtime1::handle_exception_id;
2121   } else {
2122     unwind_id = Runtime1::handle_exception_nofpu_id;
2123   }
2124   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2125 
2126   // FIXME: enough room for two byte trap   ????
2127   __ nop();
2128 }
2129 
2130 
2131 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2132   assert(exceptionOop->as_register() == r0, "must match");
2133 
2134   __ b(_unwind_handler_entry);
2135 }
2136 
2137 
2138 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2139   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2140   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2141 
2142   switch (left->type()) {
2143     case T_INT: {
2144       switch (code) {
2145       case lir_shl:  __ lslvw (dreg, lreg, count->as_register()); break;
2146       case lir_shr:  __ asrvw (dreg, lreg, count->as_register()); break;
2147       case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2148       default:
2149         ShouldNotReachHere();
2150         break;
2151       }
2152       break;
2153     case T_LONG:
2154     case T_ADDRESS:
2155     case T_OBJECT:
2156       switch (code) {
2157       case lir_shl:  __ lslv (dreg, lreg, count->as_register()); break;
2158       case lir_shr:  __ asrv (dreg, lreg, count->as_register()); break;
2159       case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2160       default:
2161         ShouldNotReachHere();
2162         break;
2163       }
2164       break;
2165     default:
2166       ShouldNotReachHere();
2167       break;
2168     }
2169   }
2170 }
2171 
2172 
2173 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2174   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2175   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2176 
2177   switch (left->type()) {
2178     case T_INT: {
2179       switch (code) {
2180       case lir_shl:  __ lslw (dreg, lreg, count); break;
2181       case lir_shr:  __ asrw (dreg, lreg, count); break;
2182       case lir_ushr: __ lsrw (dreg, lreg, count); break;
2183       default:
2184         ShouldNotReachHere();
2185         break;
2186       }
2187       break;
2188     case T_LONG:
2189     case T_ADDRESS:
2190     case T_OBJECT:
2191       switch (code) {
2192       case lir_shl:  __ lsl (dreg, lreg, count); break;
2193       case lir_shr:  __ asr (dreg, lreg, count); break;
2194       case lir_ushr: __ lsr (dreg, lreg, count); break;
2195       default:
2196         ShouldNotReachHere();
2197         break;
2198       }
2199       break;
2200     default:
2201       ShouldNotReachHere();
2202       break;
2203     }
2204   }
2205 }
2206 
2207 
2208 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2209   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2210   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2211   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2212   __ str (r, Address(sp, offset_from_rsp_in_bytes));
2213 }
2214 
2215 
2216 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2217   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2218   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2219   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2220   __ mov (rscratch1, c);
2221   __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2222 }
2223 
2224 
2225 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2226   ShouldNotReachHere();
2227   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2228   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2229   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2230   __ lea(rscratch1, __ constant_oop_address(o));
2231   __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2232 }
2233 
2234 
2235 // This code replaces a call to arraycopy; no exception may
2236 // be thrown in this code, they must be thrown in the System.arraycopy
2237 // activation frame; we could save some checks if this would not be the case
2238 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2239   ciArrayKlass* default_type = op->expected_type();
2240   Register src = op->src()->as_register();
2241   Register dst = op->dst()->as_register();
2242   Register src_pos = op->src_pos()->as_register();
2243   Register dst_pos = op->dst_pos()->as_register();
2244   Register length  = op->length()->as_register();
2245   Register tmp = op->tmp()->as_register();
2246 
2247   CodeStub* stub = op->stub();
2248   int flags = op->flags();
2249   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2250   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2251 
2252   // if we don't know anything, just go through the generic arraycopy
2253   if (default_type == NULL // || basic_type == T_OBJECT
2254       ) {
2255     Label done;
2256     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2257 
2258     // Save the arguments in case the generic arraycopy fails and we
2259     // have to fall back to the JNI stub
2260     __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2261     __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2262     __ str(src,              Address(sp, 4*BytesPerWord));
2263 
2264     address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2265     address copyfunc_addr = StubRoutines::generic_arraycopy();
2266 
2267     // The arguments are in java calling convention so we shift them
2268     // to C convention
2269     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2270     __ mov(c_rarg0, j_rarg0);
2271     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2272     __ mov(c_rarg1, j_rarg1);
2273     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2274     __ mov(c_rarg2, j_rarg2);
2275     assert_different_registers(c_rarg3, j_rarg4);
2276     __ mov(c_rarg3, j_rarg3);
2277     __ mov(c_rarg4, j_rarg4);
2278     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2279       __ mov(rscratch1, RuntimeAddress(C_entry));
2280       __ blr(rscratch1);
2281     } else {
2282 #ifndef PRODUCT
2283       if (PrintC1Statistics) {
2284         __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2285       }
2286 #endif
2287       __ far_call(RuntimeAddress(copyfunc_addr));
2288     }
2289 
2290     __ cbz(r0, *stub->continuation());
2291 
2292     // Reload values from the stack so they are where the stub
2293     // expects them.
2294     __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2295     __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2296     __ ldr(src,              Address(sp, 4*BytesPerWord));
2297 
2298     if (copyfunc_addr != NULL) {
2299       // r0 is -1^K where K == partial copied count
2300       __ eonw(rscratch1, r0, zr);
2301       // adjust length down and src/end pos up by partial copied count
2302       __ subw(length, length, rscratch1);
2303       __ addw(src_pos, src_pos, rscratch1);
2304       __ addw(dst_pos, dst_pos, rscratch1);
2305     }
2306     __ b(*stub->entry());
2307 
2308     __ bind(*stub->continuation());
2309     return;
2310   }
2311 
2312   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2313 
2314   int elem_size = type2aelembytes(basic_type);
2315   int shift_amount;
2316   int scale = exact_log2(elem_size);
2317 
2318   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2319   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2320   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2321   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2322 
2323   // test for NULL
2324   if (flags & LIR_OpArrayCopy::src_null_check) {
2325     __ cbz(src, *stub->entry());
2326   }
2327   if (flags & LIR_OpArrayCopy::dst_null_check) {
2328     __ cbz(dst, *stub->entry());
2329   }
2330 
2331   // If the compiler was not able to prove that exact type of the source or the destination
2332   // of the arraycopy is an array type, check at runtime if the source or the destination is
2333   // an instance type.
2334   if (flags & LIR_OpArrayCopy::type_check) {
2335     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2336       __ load_klass(tmp, dst);
2337       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2338       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2339       __ br(Assembler::GE, *stub->entry());
2340     }
2341 
2342     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2343       __ load_klass(tmp, src);
2344       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2345       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2346       __ br(Assembler::GE, *stub->entry());
2347     }
2348   }
2349 
2350   // check if negative
2351   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2352     __ cmpw(src_pos, 0);
2353     __ br(Assembler::LT, *stub->entry());
2354   }
2355   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2356     __ cmpw(dst_pos, 0);
2357     __ br(Assembler::LT, *stub->entry());
2358   }
2359 
2360   if (flags & LIR_OpArrayCopy::length_positive_check) {
2361     __ cmpw(length, 0);
2362     __ br(Assembler::LT, *stub->entry());
2363   }
2364 
2365   if (flags & LIR_OpArrayCopy::src_range_check) {
2366     __ addw(tmp, src_pos, length);
2367     __ ldrw(rscratch1, src_length_addr);
2368     __ cmpw(tmp, rscratch1);
2369     __ br(Assembler::HI, *stub->entry());
2370   }
2371   if (flags & LIR_OpArrayCopy::dst_range_check) {
2372     __ addw(tmp, dst_pos, length);
2373     __ ldrw(rscratch1, dst_length_addr);
2374     __ cmpw(tmp, rscratch1);
2375     __ br(Assembler::HI, *stub->entry());
2376   }
2377 
2378   // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2379   // length_positive_check if the source of our length operand is an
2380   // arraylength.  However, that arraylength might be zero, and the
2381   // stub that we're about to call contains an assertion that count !=
2382   // 0 .  So we make this check purely in order not to trigger an
2383   // assertion failure.
2384   __ cbzw(length, *stub->continuation());
2385 
2386   if (flags & LIR_OpArrayCopy::type_check) {
2387     // We don't know the array types are compatible
2388     if (basic_type != T_OBJECT) {
2389       // Simple test for basic type arrays
2390       if (UseCompressedClassPointers) {
2391         __ ldrw(tmp, src_klass_addr);
2392         __ ldrw(rscratch1, dst_klass_addr);
2393         __ cmpw(tmp, rscratch1);
2394       } else {
2395         __ ldr(tmp, src_klass_addr);
2396         __ ldr(rscratch1, dst_klass_addr);
2397         __ cmp(tmp, rscratch1);
2398       }
2399       __ br(Assembler::NE, *stub->entry());
2400     } else {
2401       // For object arrays, if src is a sub class of dst then we can
2402       // safely do the copy.
2403       Label cont, slow;
2404 
2405 #define PUSH(r1, r2)                                    \
2406       stp(r1, r2, __ pre(sp, -2 * wordSize));
2407 
2408 #define POP(r1, r2)                                     \
2409       ldp(r1, r2, __ post(sp, 2 * wordSize));
2410 
2411       __ PUSH(src, dst);
2412 
2413       __ load_klass(src, src);
2414       __ load_klass(dst, dst);
2415 
2416       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2417 
2418       __ PUSH(src, dst);
2419       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2420       __ POP(src, dst);
2421 
2422       __ cbnz(src, cont);
2423 
2424       __ bind(slow);
2425       __ POP(src, dst);
2426 
2427       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2428       if (copyfunc_addr != NULL) { // use stub if available
2429         // src is not a sub class of dst so we have to do a
2430         // per-element check.
2431 
2432         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2433         if ((flags & mask) != mask) {
2434           // Check that at least both of them object arrays.
2435           assert(flags & mask, "one of the two should be known to be an object array");
2436 
2437           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2438             __ load_klass(tmp, src);
2439           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2440             __ load_klass(tmp, dst);
2441           }
2442           int lh_offset = in_bytes(Klass::layout_helper_offset());
2443           Address klass_lh_addr(tmp, lh_offset);
2444           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2445           __ ldrw(rscratch1, klass_lh_addr);
2446           __ mov(rscratch2, objArray_lh);
2447           __ eorw(rscratch1, rscratch1, rscratch2);
2448           __ cbnzw(rscratch1, *stub->entry());
2449         }
2450 
2451        // Spill because stubs can use any register they like and it's
2452        // easier to restore just those that we care about.
2453         __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2454         __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2455         __ str(src,              Address(sp, 4*BytesPerWord));
2456 
2457         __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2458         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2459         assert_different_registers(c_rarg0, dst, dst_pos, length);
2460         __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2461         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2462         assert_different_registers(c_rarg1, dst, length);
2463         __ uxtw(c_rarg2, length);
2464         assert_different_registers(c_rarg2, dst);
2465 
2466         __ load_klass(c_rarg4, dst);
2467         __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2468         __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2469         __ far_call(RuntimeAddress(copyfunc_addr));
2470 
2471 #ifndef PRODUCT
2472         if (PrintC1Statistics) {
2473           Label failed;
2474           __ cbnz(r0, failed);
2475           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2476           __ bind(failed);
2477         }
2478 #endif
2479 
2480         __ cbz(r0, *stub->continuation());
2481 
2482 #ifndef PRODUCT
2483         if (PrintC1Statistics) {
2484           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2485         }
2486 #endif
2487         assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2488 
2489         // Restore previously spilled arguments
2490         __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2491         __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2492         __ ldr(src,              Address(sp, 4*BytesPerWord));
2493 
2494         // return value is -1^K where K is partial copied count
2495         __ eonw(rscratch1, r0, zr);
2496         // adjust length down and src/end pos up by partial copied count
2497         __ subw(length, length, rscratch1);
2498         __ addw(src_pos, src_pos, rscratch1);
2499         __ addw(dst_pos, dst_pos, rscratch1);
2500       }
2501 
2502       __ b(*stub->entry());
2503 
2504       __ bind(cont);
2505       __ POP(src, dst);
2506     }
2507   }
2508 
2509 #ifdef ASSERT
2510   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2511     // Sanity check the known type with the incoming class.  For the
2512     // primitive case the types must match exactly with src.klass and
2513     // dst.klass each exactly matching the default type.  For the
2514     // object array case, if no type check is needed then either the
2515     // dst type is exactly the expected type and the src type is a
2516     // subtype which we can't check or src is the same array as dst
2517     // but not necessarily exactly of type default_type.
2518     Label known_ok, halt;
2519     __ mov_metadata(tmp, default_type->constant_encoding());
2520 #ifdef _LP64
2521     if (UseCompressedClassPointers) {
2522       __ encode_klass_not_null(tmp);
2523     }
2524 #endif
2525 
2526     if (basic_type != T_OBJECT) {
2527 
2528       if (UseCompressedClassPointers) {
2529         __ ldrw(rscratch1, dst_klass_addr);
2530         __ cmpw(tmp, rscratch1);
2531       } else {
2532         __ ldr(rscratch1, dst_klass_addr);
2533         __ cmp(tmp, rscratch1);
2534       }
2535       __ br(Assembler::NE, halt);
2536       if (UseCompressedClassPointers) {
2537         __ ldrw(rscratch1, src_klass_addr);
2538         __ cmpw(tmp, rscratch1);
2539       } else {
2540         __ ldr(rscratch1, src_klass_addr);
2541         __ cmp(tmp, rscratch1);
2542       }
2543       __ br(Assembler::EQ, known_ok);
2544     } else {
2545       if (UseCompressedClassPointers) {
2546         __ ldrw(rscratch1, dst_klass_addr);
2547         __ cmpw(tmp, rscratch1);
2548       } else {
2549         __ ldr(rscratch1, dst_klass_addr);
2550         __ cmp(tmp, rscratch1);
2551       }
2552       __ br(Assembler::EQ, known_ok);
2553       __ cmp(src, dst);
2554       __ br(Assembler::EQ, known_ok);
2555     }
2556     __ bind(halt);
2557     __ stop("incorrect type information in arraycopy");
2558     __ bind(known_ok);
2559   }
2560 #endif
2561 
2562 #ifndef PRODUCT
2563   if (PrintC1Statistics) {
2564     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2565   }
2566 #endif
2567 
2568   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2569   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2570   assert_different_registers(c_rarg0, dst, dst_pos, length);
2571   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2572   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2573   assert_different_registers(c_rarg1, dst, length);
2574   __ uxtw(c_rarg2, length);
2575   assert_different_registers(c_rarg2, dst);
2576 
2577   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2578   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2579   const char *name;
2580   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2581 
2582  CodeBlob *cb = CodeCache::find_blob(entry);
2583  if (cb) {
2584    __ far_call(RuntimeAddress(entry));
2585  } else {
2586    __ call_VM_leaf(entry, 3);
2587  }
2588 
2589   __ bind(*stub->continuation());
2590 }
2591 
2592 
2593 
2594 
2595 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2596   Register obj = op->obj_opr()->as_register();  // may not be an oop
2597   Register hdr = op->hdr_opr()->as_register();
2598   Register lock = op->lock_opr()->as_register();
2599   if (!UseFastLocking) {
2600     __ b(*op->stub()->entry());
2601   } else if (op->code() == lir_lock) {
2602     Register scratch = noreg;
2603     if (UseBiasedLocking) {
2604       scratch = op->scratch_opr()->as_register();
2605     }
2606     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2607     // add debug info for NullPointerException only if one is possible
2608     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2609     if (op->info() != NULL) {
2610       add_debug_info_for_null_check(null_check_offset, op->info());
2611     }
2612     // done
2613   } else if (op->code() == lir_unlock) {
2614     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2615     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2616   } else {
2617     Unimplemented();
2618   }
2619   __ bind(*op->stub()->continuation());
2620 }
2621 
2622 
2623 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2624   ciMethod* method = op->profiled_method();
2625   int bci          = op->profiled_bci();
2626   ciMethod* callee = op->profiled_callee();
2627 
2628   // Update counter for all call types
2629   ciMethodData* md = method->method_data_or_null();
2630   assert(md != NULL, "Sanity");
2631   ciProfileData* data = md->bci_to_data(bci);
2632   assert(data->is_CounterData(), "need CounterData for calls");
2633   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2634   Register mdo  = op->mdo()->as_register();
2635   __ mov_metadata(mdo, md->constant_encoding());
2636   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2637   Bytecodes::Code bc = method->java_code_at_bci(bci);
2638   const bool callee_is_static = callee->is_loaded() && callee->is_static();
2639   // Perform additional virtual call profiling for invokevirtual and
2640   // invokeinterface bytecodes
2641   if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2642       !callee_is_static &&  // required for optimized MH invokes
2643       C1ProfileVirtualCalls) {
2644     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2645     Register recv = op->recv()->as_register();
2646     assert_different_registers(mdo, recv);
2647     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2648     ciKlass* known_klass = op->known_holder();
2649     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2650       // We know the type that will be seen at this call site; we can
2651       // statically update the MethodData* rather than needing to do
2652       // dynamic tests on the receiver type
2653 
2654       // NOTE: we should probably put a lock around this search to
2655       // avoid collisions by concurrent compilations
2656       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2657       uint i;
2658       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2659         ciKlass* receiver = vc_data->receiver(i);
2660         if (known_klass->equals(receiver)) {
2661           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2662           __ addptr(data_addr, DataLayout::counter_increment);
2663           return;
2664         }
2665       }
2666 
2667       // Receiver type not found in profile data; select an empty slot
2668 
2669       // Note that this is less efficient than it should be because it
2670       // always does a write to the receiver part of the
2671       // VirtualCallData rather than just the first time
2672       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2673         ciKlass* receiver = vc_data->receiver(i);
2674         if (receiver == NULL) {
2675           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2676           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2677           __ lea(rscratch2, recv_addr);
2678           __ str(rscratch1, Address(rscratch2));
2679           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2680           __ addptr(data_addr, DataLayout::counter_increment);
2681           return;
2682         }
2683       }
2684     } else {
2685       __ load_klass(recv, recv);
2686       Label update_done;
2687       type_profile_helper(mdo, md, data, recv, &update_done);
2688       // Receiver did not match any saved receiver and there is no empty row for it.
2689       // Increment total counter to indicate polymorphic case.
2690       __ addptr(counter_addr, DataLayout::counter_increment);
2691 
2692       __ bind(update_done);
2693     }
2694   } else {
2695     // Static call
2696     __ addptr(counter_addr, DataLayout::counter_increment);
2697   }
2698 }
2699 
2700 
2701 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2702   Unimplemented();
2703 }
2704 
2705 
2706 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2707   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2708 }
2709 
2710 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2711   assert(op->crc()->is_single_cpu(),  "crc must be register");
2712   assert(op->val()->is_single_cpu(),  "byte value must be register");
2713   assert(op->result_opr()->is_single_cpu(), "result must be register");
2714   Register crc = op->crc()->as_register();
2715   Register val = op->val()->as_register();
2716   Register res = op->result_opr()->as_register();
2717 
2718   assert_different_registers(val, crc, res);
2719   unsigned long offset;
2720   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2721   if (offset) __ add(res, res, offset);
2722 
2723   __ ornw(crc, zr, crc); // ~crc
2724   __ update_byte_crc32(crc, val, res);
2725   __ ornw(res, zr, crc); // ~crc
2726 }
2727 
2728 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2729   COMMENT("emit_profile_type {");
2730   Register obj = op->obj()->as_register();
2731   Register tmp = op->tmp()->as_pointer_register();
2732   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2733   ciKlass* exact_klass = op->exact_klass();
2734   intptr_t current_klass = op->current_klass();
2735   bool not_null = op->not_null();
2736   bool no_conflict = op->no_conflict();
2737 
2738   Label update, next, none;
2739 
2740   bool do_null = !not_null;
2741   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2742   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2743 
2744   assert(do_null || do_update, "why are we here?");
2745   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2746   assert(mdo_addr.base() != rscratch1, "wrong register");
2747 
2748   __ verify_oop(obj);
2749 
2750   if (tmp != obj) {
2751     __ mov(tmp, obj);
2752   }
2753   if (do_null) {
2754     __ cbnz(tmp, update);
2755     if (!TypeEntries::was_null_seen(current_klass)) {
2756       __ ldr(rscratch2, mdo_addr);
2757       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2758       __ str(rscratch2, mdo_addr);
2759     }
2760     if (do_update) {
2761 #ifndef ASSERT
2762       __ b(next);
2763     }
2764 #else
2765       __ b(next);
2766     }
2767   } else {
2768     __ cbnz(tmp, update);
2769     __ stop("unexpected null obj");
2770 #endif
2771   }
2772 
2773   __ bind(update);
2774 
2775   if (do_update) {
2776 #ifdef ASSERT
2777     if (exact_klass != NULL) {
2778       Label ok;
2779       __ load_klass(tmp, tmp);
2780       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2781       __ eor(rscratch1, tmp, rscratch1);
2782       __ cbz(rscratch1, ok);
2783       __ stop("exact klass and actual klass differ");
2784       __ bind(ok);
2785     }
2786 #endif
2787     if (!no_conflict) {
2788       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2789         if (exact_klass != NULL) {
2790           __ mov_metadata(tmp, exact_klass->constant_encoding());
2791         } else {
2792           __ load_klass(tmp, tmp);
2793         }
2794 
2795         __ ldr(rscratch2, mdo_addr);
2796         __ eor(tmp, tmp, rscratch2);
2797         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2798         // klass seen before, nothing to do. The unknown bit may have been
2799         // set already but no need to check.
2800         __ cbz(rscratch1, next);
2801 
2802         __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2803         __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2804 
2805         if (TypeEntries::is_type_none(current_klass)) {
2806           __ cbz(rscratch2, none);
2807           __ cmp(rscratch2, TypeEntries::null_seen);
2808           __ br(Assembler::EQ, none);
2809           // There is a chance that the checks above (re-reading profiling
2810           // data from memory) fail if another thread has just set the
2811           // profiling to this obj's klass
2812           __ dmb(Assembler::ISHLD);
2813           __ ldr(rscratch2, mdo_addr);
2814           __ eor(tmp, tmp, rscratch2);
2815           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2816           __ cbz(rscratch1, next);
2817         }
2818       } else {
2819         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2820                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2821 
2822         __ ldr(tmp, mdo_addr);
2823         __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2824         __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2825       }
2826 
2827       // different than before. Cannot keep accurate profile.
2828       __ ldr(rscratch2, mdo_addr);
2829       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2830       __ str(rscratch2, mdo_addr);
2831 
2832       if (TypeEntries::is_type_none(current_klass)) {
2833         __ b(next);
2834 
2835         __ bind(none);
2836         // first time here. Set profile type.
2837         __ str(tmp, mdo_addr);
2838       }
2839     } else {
2840       // There's a single possible klass at this profile point
2841       assert(exact_klass != NULL, "should be");
2842       if (TypeEntries::is_type_none(current_klass)) {
2843         __ mov_metadata(tmp, exact_klass->constant_encoding());
2844         __ ldr(rscratch2, mdo_addr);
2845         __ eor(tmp, tmp, rscratch2);
2846         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2847         __ cbz(rscratch1, next);
2848 #ifdef ASSERT
2849         {
2850           Label ok;
2851           __ ldr(rscratch1, mdo_addr);
2852           __ cbz(rscratch1, ok);
2853           __ cmp(rscratch1, TypeEntries::null_seen);
2854           __ br(Assembler::EQ, ok);
2855           // may have been set by another thread
2856           __ dmb(Assembler::ISHLD);
2857           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2858           __ ldr(rscratch2, mdo_addr);
2859           __ eor(rscratch2, rscratch1, rscratch2);
2860           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2861           __ cbz(rscratch2, ok);
2862 
2863           __ stop("unexpected profiling mismatch");
2864           __ bind(ok);
2865         }
2866 #endif
2867         // first time here. Set profile type.
2868         __ ldr(tmp, mdo_addr);
2869       } else {
2870         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2871                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2872 
2873         __ ldr(tmp, mdo_addr);
2874         __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2875         __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2876 
2877         __ orr(tmp, tmp, TypeEntries::type_unknown);
2878         __ str(tmp, mdo_addr);
2879         // FIXME: Write barrier needed here?
2880       }
2881     }
2882 
2883     __ bind(next);
2884   }
2885   COMMENT("} emit_profile_type");
2886 }
2887 
2888 
2889 void LIR_Assembler::align_backward_branch_target() {
2890 }
2891 
2892 
2893 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2894   if (left->is_single_cpu()) {
2895     assert(dest->is_single_cpu(), "expect single result reg");
2896     __ negw(dest->as_register(), left->as_register());
2897   } else if (left->is_double_cpu()) {
2898     assert(dest->is_double_cpu(), "expect double result reg");
2899     __ neg(dest->as_register_lo(), left->as_register_lo());
2900   } else if (left->is_single_fpu()) {
2901     assert(dest->is_single_fpu(), "expect single float result reg");
2902     __ fnegs(dest->as_float_reg(), left->as_float_reg());
2903   } else {
2904     assert(left->is_double_fpu(), "expect double float operand reg");
2905     assert(dest->is_double_fpu(), "expect double float result reg");
2906     __ fnegd(dest->as_double_reg(), left->as_double_reg());
2907   }
2908 }
2909 
2910 
2911 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2912 #if INCLUDE_ALL_GCS
2913   if (UseShenandoahGC && patch_code != lir_patch_none) {
2914     deoptimize_trap(info);
2915     return;
2916   }
2917 #endif
2918 
2919   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2920 }
2921 
2922 
2923 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2924   assert(!tmp->is_valid(), "don't need temporary");
2925 
2926   CodeBlob *cb = CodeCache::find_blob(dest);
2927   if (cb) {
2928     __ far_call(RuntimeAddress(dest));
2929   } else {
2930     __ mov(rscratch1, RuntimeAddress(dest));
2931     __ blr(rscratch1);
2932   }
2933 
2934   if (info != NULL) {
2935     add_call_info_here(info);
2936   }
2937   __ maybe_isb();
2938 }
2939 
2940 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2941   if (dest->is_address() || src->is_address()) {
2942     move_op(src, dest, type, lir_patch_none, info,
2943             /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2944   } else {
2945     ShouldNotReachHere();
2946   }
2947 }
2948 
2949 #ifdef ASSERT
2950 // emit run-time assertion
2951 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2952   assert(op->code() == lir_assert, "must be");
2953 
2954   if (op->in_opr1()->is_valid()) {
2955     assert(op->in_opr2()->is_valid(), "both operands must be valid");
2956     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2957   } else {
2958     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2959     assert(op->condition() == lir_cond_always, "no other conditions allowed");
2960   }
2961 
2962   Label ok;
2963   if (op->condition() != lir_cond_always) {
2964     Assembler::Condition acond = Assembler::AL;
2965     switch (op->condition()) {
2966       case lir_cond_equal:        acond = Assembler::EQ;  break;
2967       case lir_cond_notEqual:     acond = Assembler::NE;  break;
2968       case lir_cond_less:         acond = Assembler::LT;  break;
2969       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
2970       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
2971       case lir_cond_greater:      acond = Assembler::GT;  break;
2972       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
2973       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
2974       default:                    ShouldNotReachHere();
2975     }
2976     __ br(acond, ok);
2977   }
2978   if (op->halt()) {
2979     const char* str = __ code_string(op->msg());
2980     __ stop(str);
2981   } else {
2982     breakpoint();
2983   }
2984   __ bind(ok);
2985 }
2986 #endif
2987 
2988 #ifndef PRODUCT
2989 #define COMMENT(x)   do { __ block_comment(x); } while (0)
2990 #else
2991 #define COMMENT(x)
2992 #endif
2993 
2994 void LIR_Assembler::membar() {
2995   COMMENT("membar");
2996   __ membar(MacroAssembler::AnyAny);
2997 }
2998 
2999 void LIR_Assembler::membar_acquire() {
3000   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3001 }
3002 
3003 void LIR_Assembler::membar_release() {
3004   __ membar(Assembler::LoadStore|Assembler::StoreStore);
3005 }
3006 
3007 void LIR_Assembler::membar_loadload() {
3008   __ membar(Assembler::LoadLoad);
3009 }
3010 
3011 void LIR_Assembler::membar_storestore() {
3012   __ membar(MacroAssembler::StoreStore);
3013 }
3014 
3015 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3016 
3017 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3018 
3019 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3020   __ mov(result_reg->as_register(), rthread);
3021 }
3022 
3023 
3024 void LIR_Assembler::peephole(LIR_List *lir) {
3025 #if 0
3026   if (tableswitch_count >= max_tableswitches)
3027     return;
3028 
3029   /*
3030     This finite-state automaton recognizes sequences of compare-and-
3031     branch instructions.  We will turn them into a tableswitch.  You
3032     could argue that C1 really shouldn't be doing this sort of
3033     optimization, but without it the code is really horrible.
3034   */
3035 
3036   enum { start_s, cmp1_s, beq_s, cmp_s } state;
3037   int first_key, last_key = -2147483648;
3038   int next_key = 0;
3039   int start_insn = -1;
3040   int last_insn = -1;
3041   Register reg = noreg;
3042   LIR_Opr reg_opr;
3043   state = start_s;
3044 
3045   LIR_OpList* inst = lir->instructions_list();
3046   for (int i = 0; i < inst->length(); i++) {
3047     LIR_Op* op = inst->at(i);
3048     switch (state) {
3049     case start_s:
3050       first_key = -1;
3051       start_insn = i;
3052       switch (op->code()) {
3053       case lir_cmp:
3054         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3055         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3056         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3057             && opr2->is_constant()
3058             && opr2->type() == T_INT) {
3059           reg_opr = opr1;
3060           reg = opr1->as_register();
3061           first_key = opr2->as_constant_ptr()->as_jint();
3062           next_key = first_key + 1;
3063           state = cmp_s;
3064           goto next_state;
3065         }
3066         break;
3067       }
3068       break;
3069     case cmp_s:
3070       switch (op->code()) {
3071       case lir_branch:
3072         if (op->as_OpBranch()->cond() == lir_cond_equal) {
3073           state = beq_s;
3074           last_insn = i;
3075           goto next_state;
3076         }
3077       }
3078       state = start_s;
3079       break;
3080     case beq_s:
3081       switch (op->code()) {
3082       case lir_cmp: {
3083         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3084         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3085         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3086             && opr1->as_register() == reg
3087             && opr2->is_constant()
3088             && opr2->type() == T_INT
3089             && opr2->as_constant_ptr()->as_jint() == next_key) {
3090           last_key = next_key;
3091           next_key++;
3092           state = cmp_s;
3093           goto next_state;
3094         }
3095       }
3096       }
3097       last_key = next_key;
3098       state = start_s;
3099       break;
3100     default:
3101       assert(false, "impossible state");
3102     }
3103     if (state == start_s) {
3104       if (first_key < last_key - 5L && reg != noreg) {
3105         {
3106           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3107           //        reg->encoding(),
3108           //        start_insn, first_key, last_key);
3109           //   for (int i = 0; i < inst->length(); i++) {
3110           //     inst->at(i)->print();
3111           //     tty->print("\n");
3112           //   }
3113           //   tty->print("\n");
3114         }
3115 
3116         struct tableswitch *sw = &switches[tableswitch_count];
3117         sw->_insn_index = start_insn, sw->_first_key = first_key,
3118           sw->_last_key = last_key, sw->_reg = reg;
3119         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3120         {
3121           // Insert the new table of branches
3122           int offset = last_insn;
3123           for (int n = first_key; n < last_key; n++) {
3124             inst->insert_before
3125               (last_insn + 1,
3126                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3127                                 inst->at(offset)->as_OpBranch()->label()));
3128             offset -= 2, i++;
3129           }
3130         }
3131         // Delete all the old compare-and-branch instructions
3132         for (int n = first_key; n < last_key; n++) {
3133           inst->remove_at(start_insn);
3134           inst->remove_at(start_insn);
3135         }
3136         // Insert the tableswitch instruction
3137         inst->insert_before(start_insn,
3138                             new LIR_Op2(lir_cmp, lir_cond_always,
3139                                         LIR_OprFact::intConst(tableswitch_count),
3140                                         reg_opr));
3141         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3142         tableswitch_count++;
3143       }
3144       reg = noreg;
3145       last_key = -2147483648;
3146     }
3147   next_state:
3148     ;
3149   }
3150 #endif
3151 }
3152 
3153 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3154   Address addr = as_Address(src->as_address_ptr());
3155   BasicType type = src->type();
3156   bool is_oop = type == T_OBJECT || type == T_ARRAY;
3157 
3158   void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3159   void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3160 
3161   switch(type) {
3162   case T_INT:
3163     xchg = &MacroAssembler::atomic_xchgalw;
3164     add = &MacroAssembler::atomic_addalw;
3165     break;
3166   case T_LONG:
3167     xchg = &MacroAssembler::atomic_xchgal;
3168     add = &MacroAssembler::atomic_addal;
3169     break;
3170   case T_OBJECT:
3171   case T_ARRAY:
3172     if (UseCompressedOops) {
3173       xchg = &MacroAssembler::atomic_xchgalw;
3174       add = &MacroAssembler::atomic_addalw;
3175     } else {
3176       xchg = &MacroAssembler::atomic_xchgal;
3177       add = &MacroAssembler::atomic_addal;
3178     }
3179     break;
3180   default:
3181     ShouldNotReachHere();
3182     xchg = &MacroAssembler::atomic_xchgal;
3183     add = &MacroAssembler::atomic_addal; // unreachable
3184   }
3185 
3186   switch (code) {
3187   case lir_xadd:
3188     {
3189       RegisterOrConstant inc;
3190       Register tmp = as_reg(tmp_op);
3191       Register dst = as_reg(dest);
3192       if (data->is_constant()) {
3193         inc = RegisterOrConstant(as_long(data));
3194         assert_different_registers(dst, addr.base(), tmp,
3195                                    rscratch1, rscratch2);
3196       } else {
3197         inc = RegisterOrConstant(as_reg(data));
3198         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3199                                    rscratch1, rscratch2);
3200       }
3201       __ lea(tmp, addr);
3202       (_masm->*add)(dst, inc, tmp);
3203       break;
3204     }
3205   case lir_xchg:
3206     {
3207       Register tmp = tmp_op->as_register();
3208       Register obj = as_reg(data);
3209       Register dst = as_reg(dest);
3210       if (is_oop && UseCompressedOops) {
3211         __ encode_heap_oop(rscratch2, obj);
3212         obj = rscratch2;
3213       }
3214       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3215       __ lea(tmp, addr);
3216       (_masm->*xchg)(dst, obj, tmp);
3217       if (is_oop && UseCompressedOops) {
3218         __ decode_heap_oop(dst);
3219       }
3220     }
3221     break;
3222   default:
3223     ShouldNotReachHere();
3224   }
3225   __ membar(__ AnyAny);
3226 }
3227 
3228 #undef __