1 /*
   2  * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Compilation.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_MacroAssembler.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArrayKlass.hpp"
  36 #include "ci/ciInstance.hpp"
  37 #include "code/compiledIC.hpp"
  38 #include "gc/shared/barrierSet.hpp"
  39 #include "gc/shared/cardTableBarrierSet.hpp"
  40 #include "gc/shared/collectedHeap.hpp"
  41 #include "nativeInst_aarch64.hpp"
  42 #include "oops/objArrayKlass.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "utilities/macros.hpp"
  46 #include "vmreg_aarch64.inline.hpp"
  47 
  48 
  49 
  50 #ifndef PRODUCT
  51 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  52 #else
  53 #define COMMENT(x)
  54 #endif
  55 
  56 NEEDS_CLEANUP // remove this definitions ?
  57 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  58 const Register SYNC_header = r0;   // synchronization header
  59 const Register SHIFT_count = r0;   // where count for shift operations must be
  60 
  61 #define __ _masm->
  62 
  63 
  64 static void select_different_registers(Register preserve,
  65                                        Register extra,
  66                                        Register &tmp1,
  67                                        Register &tmp2) {
  68   if (tmp1 == preserve) {
  69     assert_different_registers(tmp1, tmp2, extra);
  70     tmp1 = extra;
  71   } else if (tmp2 == preserve) {
  72     assert_different_registers(tmp1, tmp2, extra);
  73     tmp2 = extra;
  74   }
  75   assert_different_registers(preserve, tmp1, tmp2);
  76 }
  77 
  78 
  79 
  80 static void select_different_registers(Register preserve,
  81                                        Register extra,
  82                                        Register &tmp1,
  83                                        Register &tmp2,
  84                                        Register &tmp3) {
  85   if (tmp1 == preserve) {
  86     assert_different_registers(tmp1, tmp2, tmp3, extra);
  87     tmp1 = extra;
  88   } else if (tmp2 == preserve) {
  89     assert_different_registers(tmp1, tmp2, tmp3, extra);
  90     tmp2 = extra;
  91   } else if (tmp3 == preserve) {
  92     assert_different_registers(tmp1, tmp2, tmp3, extra);
  93     tmp3 = extra;
  94   }
  95   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  96 }
  97 
  98 
  99 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
 100 
 101 
 102 LIR_Opr LIR_Assembler::receiverOpr() {
 103   return FrameMap::receiver_opr;
 104 }
 105 
 106 LIR_Opr LIR_Assembler::osrBufferPointer() {
 107   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 108 }
 109 
 110 //--------------fpu register translations-----------------------
 111 
 112 
 113 address LIR_Assembler::float_constant(float f) {
 114   address const_addr = __ float_constant(f);
 115   if (const_addr == NULL) {
 116     bailout("const section overflow");
 117     return __ code()->consts()->start();
 118   } else {
 119     return const_addr;
 120   }
 121 }
 122 
 123 
 124 address LIR_Assembler::double_constant(double d) {
 125   address const_addr = __ double_constant(d);
 126   if (const_addr == NULL) {
 127     bailout("const section overflow");
 128     return __ code()->consts()->start();
 129   } else {
 130     return const_addr;
 131   }
 132 }
 133 
 134 address LIR_Assembler::int_constant(jlong n) {
 135   address const_addr = __ long_constant(n);
 136   if (const_addr == NULL) {
 137     bailout("const section overflow");
 138     return __ code()->consts()->start();
 139   } else {
 140     return const_addr;
 141   }
 142 }
 143 
 144 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
 145 
 146 void LIR_Assembler::reset_FPU() { Unimplemented(); }
 147 
 148 void LIR_Assembler::fpop() { Unimplemented(); }
 149 
 150 void LIR_Assembler::fxch(int i) { Unimplemented(); }
 151 
 152 void LIR_Assembler::fld(int i) { Unimplemented(); }
 153 
 154 void LIR_Assembler::ffree(int i) { Unimplemented(); }
 155 
 156 void LIR_Assembler::breakpoint() { Unimplemented(); }
 157 
 158 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 159 
 160 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 161 
 162 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
 163 //-------------------------------------------
 164 
 165 static Register as_reg(LIR_Opr op) {
 166   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 167 }
 168 
 169 static jlong as_long(LIR_Opr data) {
 170   jlong result;
 171   switch (data->type()) {
 172   case T_INT:
 173     result = (data->as_jint());
 174     break;
 175   case T_LONG:
 176     result = (data->as_jlong());
 177     break;
 178   default:
 179     ShouldNotReachHere();
 180     result = 0;  // unreachable
 181   }
 182   return result;
 183 }
 184 
 185 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 186   Register base = addr->base()->as_pointer_register();
 187   LIR_Opr opr = addr->index();
 188   if (opr->is_cpu_register()) {
 189     Register index;
 190     if (opr->is_single_cpu())
 191       index = opr->as_register();
 192     else
 193       index = opr->as_register_lo();
 194     assert(addr->disp() == 0, "must be");
 195     switch(opr->type()) {
 196       case T_INT:
 197         return Address(base, index, Address::sxtw(addr->scale()));
 198       case T_LONG:
 199         return Address(base, index, Address::lsl(addr->scale()));
 200       default:
 201         ShouldNotReachHere();
 202       }
 203   } else  {
 204     intptr_t addr_offset = intptr_t(addr->disp());
 205     if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
 206       return Address(base, addr_offset, Address::lsl(addr->scale()));
 207     else {
 208       __ mov(tmp, addr_offset);
 209       return Address(base, tmp, Address::lsl(addr->scale()));
 210     }
 211   }
 212   return Address();
 213 }
 214 
 215 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 216   ShouldNotReachHere();
 217   return Address();
 218 }
 219 
 220 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 221   return as_Address(addr, rscratch1);
 222 }
 223 
 224 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 225   return as_Address(addr, rscratch1);  // Ouch
 226   // FIXME: This needs to be much more clever.  See x86.
 227 }
 228 
 229 
 230 void LIR_Assembler::osr_entry() {
 231   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 232   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 233   ValueStack* entry_state = osr_entry->state();
 234   int number_of_locks = entry_state->locks_size();
 235 
 236   // we jump here if osr happens with the interpreter
 237   // state set up to continue at the beginning of the
 238   // loop that triggered osr - in particular, we have
 239   // the following registers setup:
 240   //
 241   // r2: osr buffer
 242   //
 243 
 244   // build frame
 245   ciMethod* m = compilation()->method();
 246   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 247 
 248   // OSR buffer is
 249   //
 250   // locals[nlocals-1..0]
 251   // monitors[0..number_of_locks]
 252   //
 253   // locals is a direct copy of the interpreter frame so in the osr buffer
 254   // so first slot in the local array is the last local from the interpreter
 255   // and last slot is local[0] (receiver) from the interpreter
 256   //
 257   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 258   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 259   // in the interpreter frame (the method lock if a sync method)
 260 
 261   // Initialize monitors in the compiled activation.
 262   //   r2: pointer to osr buffer
 263   //
 264   // All other registers are dead at this point and the locals will be
 265   // copied into place by code emitted in the IR.
 266 
 267   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 268   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 269     int monitor_offset = BytesPerWord * method()->max_locals() +
 270       (2 * BytesPerWord) * (number_of_locks - 1);
 271     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 272     // the OSR buffer using 2 word entries: first the lock and then
 273     // the oop.
 274     for (int i = 0; i < number_of_locks; i++) {
 275       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 276 #ifdef ASSERT
 277       // verify the interpreter's monitor has a non-null object
 278       {
 279         Label L;
 280         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 281         __ cbnz(rscratch1, L);
 282         __ stop("locked object is NULL");
 283         __ bind(L);
 284       }
 285 #endif
 286       __ ldr(r19, Address(OSR_buf, slot_offset + 0));
 287       __ str(r19, frame_map()->address_for_monitor_lock(i));
 288       __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 289       __ str(r19, frame_map()->address_for_monitor_object(i));
 290     }
 291   }
 292 }
 293 
 294 
 295 // inline cache check; done before the frame is built.
 296 int LIR_Assembler::check_icache() {
 297   Register receiver = FrameMap::receiver_opr->as_register();
 298   Register ic_klass = IC_Klass;
 299   int start_offset = __ offset();
 300   __ inline_cache_check(receiver, ic_klass);
 301 
 302   // if icache check fails, then jump to runtime routine
 303   // Note: RECEIVER must still contain the receiver!
 304   Label dont;
 305   __ br(Assembler::EQ, dont);
 306   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 307 
 308   // We align the verified entry point unless the method body
 309   // (including its inline cache check) will fit in a single 64-byte
 310   // icache line.
 311   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 312     // force alignment after the cache check.
 313     __ align(CodeEntryAlignment);
 314   }
 315 
 316   __ bind(dont);
 317   return start_offset;
 318 }
 319 
 320 
 321 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 322   if (o == NULL) {
 323     __ mov(reg, zr);
 324   } else {
 325     __ movoop(reg, o, /*immediate*/true);
 326   }
 327 }
 328 
 329 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 330   address target = NULL;
 331   relocInfo::relocType reloc_type = relocInfo::none;
 332 
 333   switch (patching_id(info)) {
 334   case PatchingStub::access_field_id:
 335     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 336     reloc_type = relocInfo::section_word_type;
 337     break;
 338   case PatchingStub::load_klass_id:
 339     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 340     reloc_type = relocInfo::metadata_type;
 341     break;
 342   case PatchingStub::load_mirror_id:
 343     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 344     reloc_type = relocInfo::oop_type;
 345     break;
 346   case PatchingStub::load_appendix_id:
 347     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 348     reloc_type = relocInfo::oop_type;
 349     break;
 350   default: ShouldNotReachHere();
 351   }
 352 
 353   __ far_call(RuntimeAddress(target));
 354   add_call_info_here(info);
 355 }
 356 
 357 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 358   deoptimize_trap(info);
 359 }
 360 
 361 
 362 // This specifies the rsp decrement needed to build the frame
 363 int LIR_Assembler::initial_frame_size_in_bytes() const {
 364   // if rounding, must let FrameMap know!
 365 
 366   // The frame_map records size in slots (32bit word)
 367 
 368   // subtract two words to account for return address and link
 369   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 370 }
 371 
 372 
 373 int LIR_Assembler::emit_exception_handler() {
 374   // if the last instruction is a call (typically to do a throw which
 375   // is coming at the end after block reordering) the return address
 376   // must still point into the code area in order to avoid assertion
 377   // failures when searching for the corresponding bci => add a nop
 378   // (was bug 5/14/1999 - gri)
 379   __ nop();
 380 
 381   // generate code for exception handler
 382   address handler_base = __ start_a_stub(exception_handler_size());
 383   if (handler_base == NULL) {
 384     // not enough space left for the handler
 385     bailout("exception handler overflow");
 386     return -1;
 387   }
 388 
 389   int offset = code_offset();
 390 
 391   // the exception oop and pc are in r0, and r3
 392   // no other registers need to be preserved, so invalidate them
 393   __ invalidate_registers(false, true, true, false, true, true);
 394 
 395   // check that there is really an exception
 396   __ verify_not_null_oop(r0);
 397 
 398   // search an exception handler (r0: exception oop, r3: throwing pc)
 399   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));  __ should_not_reach_here();
 400   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 401   __ end_a_stub();
 402 
 403   return offset;
 404 }
 405 
 406 
 407 // Emit the code to remove the frame from the stack in the exception
 408 // unwind path.
 409 int LIR_Assembler::emit_unwind_handler() {
 410 #ifndef PRODUCT
 411   if (CommentedAssembly) {
 412     _masm->block_comment("Unwind handler");
 413   }
 414 #endif
 415 
 416   int offset = code_offset();
 417 
 418   // Fetch the exception from TLS and clear out exception related thread state
 419   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 420   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 421   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 422 
 423   __ bind(_unwind_handler_entry);
 424   __ verify_not_null_oop(r0);
 425   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 426     __ mov(r19, r0);  // Preserve the exception
 427   }
 428 
 429   // Preform needed unlocking
 430   MonitorExitStub* stub = NULL;
 431   if (method()->is_synchronized()) {
 432     monitor_address(0, FrameMap::r0_opr);
 433     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
 434     __ unlock_object(r5, r4, r0, *stub->entry());
 435     __ bind(*stub->continuation());
 436   }
 437 
 438   if (compilation()->env()->dtrace_method_probes()) {
 439     __ call_Unimplemented();
 440 #if 0
 441     __ movptr(Address(rsp, 0), rax);
 442     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
 443     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 444 #endif
 445   }
 446 
 447   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 448     __ mov(r0, r19);  // Restore the exception
 449   }
 450 
 451   // remove the activation and dispatch to the unwind handler
 452   __ block_comment("remove_frame and dispatch to the unwind handler");
 453   __ remove_frame(initial_frame_size_in_bytes());
 454   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 455 
 456   // Emit the slow path assembly
 457   if (stub != NULL) {
 458     stub->emit_code(this);
 459   }
 460 
 461   return offset;
 462 }
 463 
 464 
 465 int LIR_Assembler::emit_deopt_handler() {
 466   // if the last instruction is a call (typically to do a throw which
 467   // is coming at the end after block reordering) the return address
 468   // must still point into the code area in order to avoid assertion
 469   // failures when searching for the corresponding bci => add a nop
 470   // (was bug 5/14/1999 - gri)
 471   __ nop();
 472 
 473   // generate code for exception handler
 474   address handler_base = __ start_a_stub(deopt_handler_size());
 475   if (handler_base == NULL) {
 476     // not enough space left for the handler
 477     bailout("deopt handler overflow");
 478     return -1;
 479   }
 480 
 481   int offset = code_offset();
 482 
 483   __ adr(lr, pc());
 484   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 485   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 486   __ end_a_stub();
 487 
 488   return offset;
 489 }
 490 
 491 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 492   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 493   int pc_offset = code_offset();
 494   flush_debug_info(pc_offset);
 495   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 496   if (info->exception_handlers() != NULL) {
 497     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 498   }
 499 }
 500 
 501 void LIR_Assembler::return_op(LIR_Opr result) {
 502   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 503 
 504   // Pop the stack before the safepoint code
 505   __ remove_frame(initial_frame_size_in_bytes());
 506 
 507   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 508     __ reserved_stack_check();
 509   }
 510 
 511   address polling_page(os::get_polling_page());
 512   __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
 513   __ ret(lr);
 514 }
 515 
 516 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 517   address polling_page(os::get_polling_page());
 518   guarantee(info != NULL, "Shouldn't be NULL");
 519   assert(os::is_poll_address(polling_page), "should be");
 520   __ get_polling_page(rscratch1, polling_page, relocInfo::poll_type);
 521   add_debug_info_for_branch(info);  // This isn't just debug info:
 522                                     // it's the oop map
 523   __ read_polling_page(rscratch1, relocInfo::poll_type);
 524   return __ offset();
 525 }
 526 
 527 
 528 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 529   if (from_reg == r31_sp)
 530     from_reg = sp;
 531   if (to_reg == r31_sp)
 532     to_reg = sp;
 533   __ mov(to_reg, from_reg);
 534 }
 535 
 536 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 537 
 538 
 539 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 540   assert(src->is_constant(), "should not call otherwise");
 541   assert(dest->is_register(), "should not call otherwise");
 542   LIR_Const* c = src->as_constant_ptr();
 543 
 544   switch (c->type()) {
 545     case T_INT: {
 546       assert(patch_code == lir_patch_none, "no patching handled here");
 547       __ movw(dest->as_register(), c->as_jint());
 548       break;
 549     }
 550 
 551     case T_ADDRESS: {
 552       assert(patch_code == lir_patch_none, "no patching handled here");
 553       __ mov(dest->as_register(), c->as_jint());
 554       break;
 555     }
 556 
 557     case T_LONG: {
 558       assert(patch_code == lir_patch_none, "no patching handled here");
 559       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
 560       break;
 561     }
 562 
 563     case T_OBJECT: {
 564         if (patch_code == lir_patch_none) {
 565           jobject2reg(c->as_jobject(), dest->as_register());
 566         } else {
 567           jobject2reg_with_patching(dest->as_register(), info);
 568         }
 569       break;
 570     }
 571 
 572     case T_METADATA: {
 573       if (patch_code != lir_patch_none) {
 574         klass2reg_with_patching(dest->as_register(), info);
 575       } else {
 576         __ mov_metadata(dest->as_register(), c->as_metadata());
 577       }
 578       break;
 579     }
 580 
 581     case T_FLOAT: {
 582       if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 583         __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
 584       } else {
 585         __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 586         __ ldrs(dest->as_float_reg(), Address(rscratch1));
 587       }
 588       break;
 589     }
 590 
 591     case T_DOUBLE: {
 592       if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
 593         __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
 594       } else {
 595         __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 596         __ ldrd(dest->as_double_reg(), Address(rscratch1));
 597       }
 598       break;
 599     }
 600 
 601     default:
 602       ShouldNotReachHere();
 603   }
 604 }
 605 
 606 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 607   LIR_Const* c = src->as_constant_ptr();
 608   switch (c->type()) {
 609   case T_OBJECT:
 610     {
 611       if (! c->as_jobject())
 612         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 613       else {
 614         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 615         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 616       }
 617     }
 618     break;
 619   case T_ADDRESS:
 620     {
 621       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 622       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 623     }
 624   case T_INT:
 625   case T_FLOAT:
 626     {
 627       Register reg = zr;
 628       if (c->as_jint_bits() == 0)
 629         __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 630       else {
 631         __ movw(rscratch1, c->as_jint_bits());
 632         __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 633       }
 634     }
 635     break;
 636   case T_LONG:
 637   case T_DOUBLE:
 638     {
 639       Register reg = zr;
 640       if (c->as_jlong_bits() == 0)
 641         __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 642                                                  lo_word_offset_in_bytes));
 643       else {
 644         __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
 645         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 646                                                         lo_word_offset_in_bytes));
 647       }
 648     }
 649     break;
 650   default:
 651     ShouldNotReachHere();
 652   }
 653 }
 654 
 655 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 656   assert(src->is_constant(), "should not call otherwise");
 657   LIR_Const* c = src->as_constant_ptr();
 658   LIR_Address* to_addr = dest->as_address_ptr();
 659 
 660   void (Assembler::* insn)(Register Rt, const Address &adr);
 661 
 662   switch (type) {
 663   case T_ADDRESS:
 664     assert(c->as_jint() == 0, "should be");
 665     insn = &Assembler::str;
 666     break;
 667   case T_LONG:
 668     assert(c->as_jlong() == 0, "should be");
 669     insn = &Assembler::str;
 670     break;
 671   case T_INT:
 672     assert(c->as_jint() == 0, "should be");
 673     insn = &Assembler::strw;
 674     break;
 675   case T_OBJECT:
 676   case T_ARRAY:
 677     assert(c->as_jobject() == 0, "should be");
 678     if (UseCompressedOops && !wide) {
 679       insn = &Assembler::strw;
 680     } else {
 681       insn = &Assembler::str;
 682     }
 683     break;
 684   case T_CHAR:
 685   case T_SHORT:
 686     assert(c->as_jint() == 0, "should be");
 687     insn = &Assembler::strh;
 688     break;
 689   case T_BOOLEAN:
 690   case T_BYTE:
 691     assert(c->as_jint() == 0, "should be");
 692     insn = &Assembler::strb;
 693     break;
 694   default:
 695     ShouldNotReachHere();
 696     insn = &Assembler::str;  // unreachable
 697   }
 698 
 699   if (info) add_debug_info_for_null_check_here(info);
 700   (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
 701 }
 702 
 703 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 704   assert(src->is_register(), "should not call otherwise");
 705   assert(dest->is_register(), "should not call otherwise");
 706 
 707   // move between cpu-registers
 708   if (dest->is_single_cpu()) {
 709     if (src->type() == T_LONG) {
 710       // Can do LONG -> OBJECT
 711       move_regs(src->as_register_lo(), dest->as_register());
 712       return;
 713     }
 714     assert(src->is_single_cpu(), "must match");
 715     if (src->type() == T_OBJECT) {
 716       __ verify_oop(src->as_register());
 717     }
 718     move_regs(src->as_register(), dest->as_register());
 719 
 720   } else if (dest->is_double_cpu()) {
 721     if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
 722       // Surprising to me but we can see move of a long to t_object
 723       __ verify_oop(src->as_register());
 724       move_regs(src->as_register(), dest->as_register_lo());
 725       return;
 726     }
 727     assert(src->is_double_cpu(), "must match");
 728     Register f_lo = src->as_register_lo();
 729     Register f_hi = src->as_register_hi();
 730     Register t_lo = dest->as_register_lo();
 731     Register t_hi = dest->as_register_hi();
 732     assert(f_hi == f_lo, "must be same");
 733     assert(t_hi == t_lo, "must be same");
 734     move_regs(f_lo, t_lo);
 735 
 736   } else if (dest->is_single_fpu()) {
 737     __ fmovs(dest->as_float_reg(), src->as_float_reg());
 738 
 739   } else if (dest->is_double_fpu()) {
 740     __ fmovd(dest->as_double_reg(), src->as_double_reg());
 741 
 742   } else {
 743     ShouldNotReachHere();
 744   }
 745 }
 746 
 747 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 748   if (src->is_single_cpu()) {
 749     if (type == T_ARRAY || type == T_OBJECT) {
 750       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 751       __ verify_oop(src->as_register());
 752     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 753       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 754     } else {
 755       __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 756     }
 757 
 758   } else if (src->is_double_cpu()) {
 759     Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 760     __ str(src->as_register_lo(), dest_addr_LO);
 761 
 762   } else if (src->is_single_fpu()) {
 763     Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 764     __ strs(src->as_float_reg(), dest_addr);
 765 
 766   } else if (src->is_double_fpu()) {
 767     Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 768     __ strd(src->as_double_reg(), dest_addr);
 769 
 770   } else {
 771     ShouldNotReachHere();
 772   }
 773 
 774 }
 775 
 776 
 777 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 778   LIR_Address* to_addr = dest->as_address_ptr();
 779   PatchingStub* patch = NULL;
 780   Register compressed_src = rscratch1;
 781 
 782   if (patch_code != lir_patch_none) {
 783     deoptimize_trap(info);
 784     return;
 785   }
 786 
 787   if (type == T_ARRAY || type == T_OBJECT) {
 788     __ verify_oop(src->as_register());
 789 
 790     if (UseCompressedOops && !wide) {
 791       __ encode_heap_oop(compressed_src, src->as_register());
 792     } else {
 793       compressed_src = src->as_register();
 794     }
 795   }
 796 
 797   int null_check_here = code_offset();
 798   switch (type) {
 799     case T_FLOAT: {
 800       __ strs(src->as_float_reg(), as_Address(to_addr));
 801       break;
 802     }
 803 
 804     case T_DOUBLE: {
 805       __ strd(src->as_double_reg(), as_Address(to_addr));
 806       break;
 807     }
 808 
 809     case T_ARRAY:   // fall through
 810     case T_OBJECT:  // fall through
 811       if (UseCompressedOops && !wide) {
 812         __ strw(compressed_src, as_Address(to_addr, rscratch2));
 813       } else {
 814          __ str(compressed_src, as_Address(to_addr));
 815       }
 816       break;
 817     case T_METADATA:
 818       // We get here to store a method pointer to the stack to pass to
 819       // a dtrace runtime call. This can't work on 64 bit with
 820       // compressed klass ptrs: T_METADATA can be a compressed klass
 821       // ptr or a 64 bit method pointer.
 822       ShouldNotReachHere();
 823       __ str(src->as_register(), as_Address(to_addr));
 824       break;
 825     case T_ADDRESS:
 826       __ str(src->as_register(), as_Address(to_addr));
 827       break;
 828     case T_INT:
 829       __ strw(src->as_register(), as_Address(to_addr));
 830       break;
 831 
 832     case T_LONG: {
 833       __ str(src->as_register_lo(), as_Address_lo(to_addr));
 834       break;
 835     }
 836 
 837     case T_BYTE:    // fall through
 838     case T_BOOLEAN: {
 839       __ strb(src->as_register(), as_Address(to_addr));
 840       break;
 841     }
 842 
 843     case T_CHAR:    // fall through
 844     case T_SHORT:
 845       __ strh(src->as_register(), as_Address(to_addr));
 846       break;
 847 
 848     default:
 849       ShouldNotReachHere();
 850   }
 851   if (info != NULL) {
 852     add_debug_info_for_null_check(null_check_here, info);
 853   }
 854 }
 855 
 856 
 857 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 858   assert(src->is_stack(), "should not call otherwise");
 859   assert(dest->is_register(), "should not call otherwise");
 860 
 861   if (dest->is_single_cpu()) {
 862     if (type == T_ARRAY || type == T_OBJECT) {
 863       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 864       __ verify_oop(dest->as_register());
 865     } else if (type == T_METADATA || type == T_ADDRESS) {
 866       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 867     } else {
 868       __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 869     }
 870 
 871   } else if (dest->is_double_cpu()) {
 872     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 873     __ ldr(dest->as_register_lo(), src_addr_LO);
 874 
 875   } else if (dest->is_single_fpu()) {
 876     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 877     __ ldrs(dest->as_float_reg(), src_addr);
 878 
 879   } else if (dest->is_double_fpu()) {
 880     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 881     __ ldrd(dest->as_double_reg(), src_addr);
 882 
 883   } else {
 884     ShouldNotReachHere();
 885   }
 886 }
 887 
 888 
 889 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 890   address target = NULL;
 891   relocInfo::relocType reloc_type = relocInfo::none;
 892 
 893   switch (patching_id(info)) {
 894   case PatchingStub::access_field_id:
 895     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 896     reloc_type = relocInfo::section_word_type;
 897     break;
 898   case PatchingStub::load_klass_id:
 899     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 900     reloc_type = relocInfo::metadata_type;
 901     break;
 902   case PatchingStub::load_mirror_id:
 903     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 904     reloc_type = relocInfo::oop_type;
 905     break;
 906   case PatchingStub::load_appendix_id:
 907     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 908     reloc_type = relocInfo::oop_type;
 909     break;
 910   default: ShouldNotReachHere();
 911   }
 912 
 913   __ far_call(RuntimeAddress(target));
 914   add_call_info_here(info);
 915 }
 916 
 917 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 918 
 919   LIR_Opr temp;
 920   if (type == T_LONG || type == T_DOUBLE)
 921     temp = FrameMap::rscratch1_long_opr;
 922   else
 923     temp = FrameMap::rscratch1_opr;
 924 
 925   stack2reg(src, temp, src->type());
 926   reg2stack(temp, dest, dest->type(), false);
 927 }
 928 
 929 
 930 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
 931   LIR_Address* addr = src->as_address_ptr();
 932   LIR_Address* from_addr = src->as_address_ptr();
 933 
 934   if (addr->base()->type() == T_OBJECT) {
 935     __ verify_oop(addr->base()->as_pointer_register());
 936   }
 937 
 938   if (patch_code != lir_patch_none) {
 939     deoptimize_trap(info);
 940     return;
 941   }
 942 
 943   if (info != NULL) {
 944     add_debug_info_for_null_check_here(info);
 945   }
 946   int null_check_here = code_offset();
 947   switch (type) {
 948     case T_FLOAT: {
 949       __ ldrs(dest->as_float_reg(), as_Address(from_addr));
 950       break;
 951     }
 952 
 953     case T_DOUBLE: {
 954       __ ldrd(dest->as_double_reg(), as_Address(from_addr));
 955       break;
 956     }
 957 
 958     case T_ARRAY:   // fall through
 959     case T_OBJECT:  // fall through
 960       if (UseCompressedOops && !wide) {
 961         __ ldrw(dest->as_register(), as_Address(from_addr));
 962       } else {
 963          __ ldr(dest->as_register(), as_Address(from_addr));
 964       }
 965       break;
 966     case T_METADATA:
 967       // We get here to store a method pointer to the stack to pass to
 968       // a dtrace runtime call. This can't work on 64 bit with
 969       // compressed klass ptrs: T_METADATA can be a compressed klass
 970       // ptr or a 64 bit method pointer.
 971       ShouldNotReachHere();
 972       __ ldr(dest->as_register(), as_Address(from_addr));
 973       break;
 974     case T_ADDRESS:
 975       // FIXME: OMG this is a horrible kludge.  Any offset from an
 976       // address that matches klass_offset_in_bytes() will be loaded
 977       // as a word, not a long.
 978       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
 979         __ ldrw(dest->as_register(), as_Address(from_addr));
 980       } else {
 981         __ ldr(dest->as_register(), as_Address(from_addr));
 982       }
 983       break;
 984     case T_INT:
 985       __ ldrw(dest->as_register(), as_Address(from_addr));
 986       break;
 987 
 988     case T_LONG: {
 989       __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
 990       break;
 991     }
 992 
 993     case T_BYTE:
 994       __ ldrsb(dest->as_register(), as_Address(from_addr));
 995       break;
 996     case T_BOOLEAN: {
 997       __ ldrb(dest->as_register(), as_Address(from_addr));
 998       break;
 999     }
1000 
1001     case T_CHAR:
1002       __ ldrh(dest->as_register(), as_Address(from_addr));
1003       break;
1004     case T_SHORT:
1005       __ ldrsh(dest->as_register(), as_Address(from_addr));
1006       break;
1007 
1008     default:
1009       ShouldNotReachHere();
1010   }
1011 
1012   if (type == T_ARRAY || type == T_OBJECT) {
1013     if (UseCompressedOops && !wide) {
1014       __ decode_heap_oop(dest->as_register());
1015     }
1016     __ verify_oop(dest->as_register());
1017   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1018     if (UseCompressedClassPointers) {
1019       __ decode_klass_not_null(dest->as_register());
1020     }
1021   }
1022 }
1023 
1024 
1025 int LIR_Assembler::array_element_size(BasicType type) const {
1026   int elem_size = type2aelembytes(type);
1027   return exact_log2(elem_size);
1028 }
1029 
1030 void LIR_Assembler::arithmetic_idiv(LIR_Op3* op, bool is_irem) {
1031   Register Rdividend = op->in_opr1()->as_register();
1032   Register Rdivisor  = op->in_opr2()->as_register();
1033   Register Rscratch  = op->in_opr3()->as_register();
1034   Register Rresult   = op->result_opr()->as_register();
1035   int divisor = -1;
1036 
1037   /*
1038   TODO: For some reason, using the Rscratch that gets passed in is
1039   not possible because the register allocator does not see the tmp reg
1040   as used, and assignes it the same register as Rdividend. We use rscratch1
1041    instead.
1042 
1043   assert(Rdividend != Rscratch, "");
1044   assert(Rdivisor  != Rscratch, "");
1045   */
1046 
1047   if (Rdivisor == noreg && is_power_of_2(divisor)) {
1048     // convert division by a power of two into some shifts and logical operations
1049   }
1050 
1051   __ corrected_idivl(Rresult, Rdividend, Rdivisor, is_irem, rscratch1);
1052 }
1053 
1054 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1055   switch (op->code()) {
1056   case lir_idiv:
1057     arithmetic_idiv(op, false);
1058     break;
1059   case lir_irem:
1060     arithmetic_idiv(op, true);
1061     break;
1062   case lir_fmad:
1063     __ fmaddd(op->result_opr()->as_double_reg(),
1064               op->in_opr1()->as_double_reg(),
1065               op->in_opr2()->as_double_reg(),
1066               op->in_opr3()->as_double_reg());
1067     break;
1068   case lir_fmaf:
1069     __ fmadds(op->result_opr()->as_float_reg(),
1070               op->in_opr1()->as_float_reg(),
1071               op->in_opr2()->as_float_reg(),
1072               op->in_opr3()->as_float_reg());
1073     break;
1074   default:      ShouldNotReachHere(); break;
1075   }
1076 }
1077 
1078 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1079 #ifdef ASSERT
1080   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1081   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1082   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1083 #endif
1084 
1085   if (op->cond() == lir_cond_always) {
1086     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1087     __ b(*(op->label()));
1088   } else {
1089     Assembler::Condition acond;
1090     if (op->code() == lir_cond_float_branch) {
1091       bool is_unordered = (op->ublock() == op->block());
1092       // Assembler::EQ does not permit unordered branches, so we add
1093       // another branch here.  Likewise, Assembler::NE does not permit
1094       // ordered branches.
1095       if ((is_unordered && op->cond() == lir_cond_equal)
1096           || (!is_unordered && op->cond() == lir_cond_notEqual))
1097         __ br(Assembler::VS, *(op->ublock()->label()));
1098       switch(op->cond()) {
1099       case lir_cond_equal:        acond = Assembler::EQ; break;
1100       case lir_cond_notEqual:     acond = Assembler::NE; break;
1101       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1102       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1103       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1104       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1105       default:                    ShouldNotReachHere();
1106         acond = Assembler::EQ;  // unreachable
1107       }
1108     } else {
1109       switch (op->cond()) {
1110         case lir_cond_equal:        acond = Assembler::EQ; break;
1111         case lir_cond_notEqual:     acond = Assembler::NE; break;
1112         case lir_cond_less:         acond = Assembler::LT; break;
1113         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1114         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1115         case lir_cond_greater:      acond = Assembler::GT; break;
1116         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1117         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1118         default:                    ShouldNotReachHere();
1119           acond = Assembler::EQ;  // unreachable
1120       }
1121     }
1122     __ br(acond,*(op->label()));
1123   }
1124 }
1125 
1126 
1127 
1128 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1129   LIR_Opr src  = op->in_opr();
1130   LIR_Opr dest = op->result_opr();
1131 
1132   switch (op->bytecode()) {
1133     case Bytecodes::_i2f:
1134       {
1135         __ scvtfws(dest->as_float_reg(), src->as_register());
1136         break;
1137       }
1138     case Bytecodes::_i2d:
1139       {
1140         __ scvtfwd(dest->as_double_reg(), src->as_register());
1141         break;
1142       }
1143     case Bytecodes::_l2d:
1144       {
1145         __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1146         break;
1147       }
1148     case Bytecodes::_l2f:
1149       {
1150         __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1151         break;
1152       }
1153     case Bytecodes::_f2d:
1154       {
1155         __ fcvts(dest->as_double_reg(), src->as_float_reg());
1156         break;
1157       }
1158     case Bytecodes::_d2f:
1159       {
1160         __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1161         break;
1162       }
1163     case Bytecodes::_i2c:
1164       {
1165         __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1166         break;
1167       }
1168     case Bytecodes::_i2l:
1169       {
1170         __ sxtw(dest->as_register_lo(), src->as_register());
1171         break;
1172       }
1173     case Bytecodes::_i2s:
1174       {
1175         __ sxth(dest->as_register(), src->as_register());
1176         break;
1177       }
1178     case Bytecodes::_i2b:
1179       {
1180         __ sxtb(dest->as_register(), src->as_register());
1181         break;
1182       }
1183     case Bytecodes::_l2i:
1184       {
1185         _masm->block_comment("FIXME: This could be a no-op");
1186         __ uxtw(dest->as_register(), src->as_register_lo());
1187         break;
1188       }
1189     case Bytecodes::_d2l:
1190       {
1191         __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1192         break;
1193       }
1194     case Bytecodes::_f2i:
1195       {
1196         __ fcvtzsw(dest->as_register(), src->as_float_reg());
1197         break;
1198       }
1199     case Bytecodes::_f2l:
1200       {
1201         __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1202         break;
1203       }
1204     case Bytecodes::_d2i:
1205       {
1206         __ fcvtzdw(dest->as_register(), src->as_double_reg());
1207         break;
1208       }
1209     default: ShouldNotReachHere();
1210   }
1211 }
1212 
1213 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1214   if (op->init_check()) {
1215     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1216                                InstanceKlass::init_state_offset()));
1217     __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1218     add_debug_info_for_null_check_here(op->stub()->info());
1219     __ br(Assembler::NE, *op->stub()->entry());
1220   }
1221   __ allocate_object(op->obj()->as_register(),
1222                      op->tmp1()->as_register(),
1223                      op->tmp2()->as_register(),
1224                      op->header_size(),
1225                      op->object_size(),
1226                      op->klass()->as_register(),
1227                      *op->stub()->entry());
1228   __ bind(*op->stub()->continuation());
1229 }
1230 
1231 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1232   Register len =  op->len()->as_register();
1233   __ uxtw(len, len);
1234 
1235   if (UseSlowPath ||
1236       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1237       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1238     __ b(*op->stub()->entry());
1239   } else {
1240     Register tmp1 = op->tmp1()->as_register();
1241     Register tmp2 = op->tmp2()->as_register();
1242     Register tmp3 = op->tmp3()->as_register();
1243     if (len == tmp1) {
1244       tmp1 = tmp3;
1245     } else if (len == tmp2) {
1246       tmp2 = tmp3;
1247     } else if (len == tmp3) {
1248       // everything is ok
1249     } else {
1250       __ mov(tmp3, len);
1251     }
1252     __ allocate_array(op->obj()->as_register(),
1253                       len,
1254                       tmp1,
1255                       tmp2,
1256                       arrayOopDesc::header_size(op->type()),
1257                       array_element_size(op->type()),
1258                       op->klass()->as_register(),
1259                       *op->stub()->entry());
1260   }
1261   __ bind(*op->stub()->continuation());
1262 }
1263 
1264 void LIR_Assembler::type_profile_helper(Register mdo,
1265                                         ciMethodData *md, ciProfileData *data,
1266                                         Register recv, Label* update_done) {
1267   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1268     Label next_test;
1269     // See if the receiver is receiver[n].
1270     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1271     __ ldr(rscratch1, Address(rscratch2));
1272     __ cmp(recv, rscratch1);
1273     __ br(Assembler::NE, next_test);
1274     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1275     __ addptr(data_addr, DataLayout::counter_increment);
1276     __ b(*update_done);
1277     __ bind(next_test);
1278   }
1279 
1280   // Didn't find receiver; find next empty slot and fill it in
1281   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1282     Label next_test;
1283     __ lea(rscratch2,
1284            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1285     Address recv_addr(rscratch2);
1286     __ ldr(rscratch1, recv_addr);
1287     __ cbnz(rscratch1, next_test);
1288     __ str(recv, recv_addr);
1289     __ mov(rscratch1, DataLayout::counter_increment);
1290     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1291     __ str(rscratch1, Address(rscratch2));
1292     __ b(*update_done);
1293     __ bind(next_test);
1294   }
1295 }
1296 
1297 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1298   // we always need a stub for the failure case.
1299   CodeStub* stub = op->stub();
1300   Register obj = op->object()->as_register();
1301   Register k_RInfo = op->tmp1()->as_register();
1302   Register klass_RInfo = op->tmp2()->as_register();
1303   Register dst = op->result_opr()->as_register();
1304   ciKlass* k = op->klass();
1305   Register Rtmp1 = noreg;
1306 
1307   // check if it needs to be profiled
1308   ciMethodData* md;
1309   ciProfileData* data;
1310 
1311   const bool should_profile = op->should_profile();
1312 
1313   if (should_profile) {
1314     ciMethod* method = op->profiled_method();
1315     assert(method != NULL, "Should have method");
1316     int bci = op->profiled_bci();
1317     md = method->method_data_or_null();
1318     assert(md != NULL, "Sanity");
1319     data = md->bci_to_data(bci);
1320     assert(data != NULL,                "need data for type check");
1321     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1322   }
1323   Label profile_cast_success, profile_cast_failure;
1324   Label *success_target = should_profile ? &profile_cast_success : success;
1325   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1326 
1327   if (obj == k_RInfo) {
1328     k_RInfo = dst;
1329   } else if (obj == klass_RInfo) {
1330     klass_RInfo = dst;
1331   }
1332   if (k->is_loaded() && !UseCompressedClassPointers) {
1333     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1334   } else {
1335     Rtmp1 = op->tmp3()->as_register();
1336     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1337   }
1338 
1339   assert_different_registers(obj, k_RInfo, klass_RInfo);
1340 
1341     if (should_profile) {
1342       Label not_null;
1343       __ cbnz(obj, not_null);
1344       // Object is null; update MDO and exit
1345       Register mdo  = klass_RInfo;
1346       __ mov_metadata(mdo, md->constant_encoding());
1347       Address data_addr
1348         = __ form_address(rscratch2, mdo,
1349                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1350                           0);
1351       __ ldrb(rscratch1, data_addr);
1352       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1353       __ strb(rscratch1, data_addr);
1354       __ b(*obj_is_null);
1355       __ bind(not_null);
1356     } else {
1357       __ cbz(obj, *obj_is_null);
1358     }
1359 
1360   if (!k->is_loaded()) {
1361     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1362   } else {
1363     __ mov_metadata(k_RInfo, k->constant_encoding());
1364   }
1365   __ verify_oop(obj);
1366 
1367   if (op->fast_check()) {
1368     // get object class
1369     // not a safepoint as obj null check happens earlier
1370     __ load_klass(rscratch1, obj);
1371     __ cmp( rscratch1, k_RInfo);
1372 
1373     __ br(Assembler::NE, *failure_target);
1374     // successful cast, fall through to profile or jump
1375   } else {
1376     // get object class
1377     // not a safepoint as obj null check happens earlier
1378     __ load_klass(klass_RInfo, obj);
1379     if (k->is_loaded()) {
1380       // See if we get an immediate positive hit
1381       __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1382       __ cmp(k_RInfo, rscratch1);
1383       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1384         __ br(Assembler::NE, *failure_target);
1385         // successful cast, fall through to profile or jump
1386       } else {
1387         // See if we get an immediate positive hit
1388         __ br(Assembler::EQ, *success_target);
1389         // check for self
1390         __ cmp(klass_RInfo, k_RInfo);
1391         __ br(Assembler::EQ, *success_target);
1392 
1393         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1394         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1395         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1396         // result is a boolean
1397         __ cbzw(klass_RInfo, *failure_target);
1398         // successful cast, fall through to profile or jump
1399       }
1400     } else {
1401       // perform the fast part of the checking logic
1402       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1403       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1404       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1405       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1406       __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1407       // result is a boolean
1408       __ cbz(k_RInfo, *failure_target);
1409       // successful cast, fall through to profile or jump
1410     }
1411   }
1412   if (should_profile) {
1413     Register mdo  = klass_RInfo, recv = k_RInfo;
1414     __ bind(profile_cast_success);
1415     __ mov_metadata(mdo, md->constant_encoding());
1416     __ load_klass(recv, obj);
1417     Label update_done;
1418     type_profile_helper(mdo, md, data, recv, success);
1419     __ b(*success);
1420 
1421     __ bind(profile_cast_failure);
1422     __ mov_metadata(mdo, md->constant_encoding());
1423     Address counter_addr
1424       = __ form_address(rscratch2, mdo,
1425                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1426                         0);
1427     __ ldr(rscratch1, counter_addr);
1428     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1429     __ str(rscratch1, counter_addr);
1430     __ b(*failure);
1431   }
1432   __ b(*success);
1433 }
1434 
1435 
1436 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1437   const bool should_profile = op->should_profile();
1438 
1439   LIR_Code code = op->code();
1440   if (code == lir_store_check) {
1441     Register value = op->object()->as_register();
1442     Register array = op->array()->as_register();
1443     Register k_RInfo = op->tmp1()->as_register();
1444     Register klass_RInfo = op->tmp2()->as_register();
1445     Register Rtmp1 = op->tmp3()->as_register();
1446 
1447     CodeStub* stub = op->stub();
1448 
1449     // check if it needs to be profiled
1450     ciMethodData* md;
1451     ciProfileData* data;
1452 
1453     if (should_profile) {
1454       ciMethod* method = op->profiled_method();
1455       assert(method != NULL, "Should have method");
1456       int bci = op->profiled_bci();
1457       md = method->method_data_or_null();
1458       assert(md != NULL, "Sanity");
1459       data = md->bci_to_data(bci);
1460       assert(data != NULL,                "need data for type check");
1461       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1462     }
1463     Label profile_cast_success, profile_cast_failure, done;
1464     Label *success_target = should_profile ? &profile_cast_success : &done;
1465     Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1466 
1467     if (should_profile) {
1468       Label not_null;
1469       __ cbnz(value, not_null);
1470       // Object is null; update MDO and exit
1471       Register mdo  = klass_RInfo;
1472       __ mov_metadata(mdo, md->constant_encoding());
1473       Address data_addr
1474         = __ form_address(rscratch2, mdo,
1475                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1476                           0);
1477       __ ldrb(rscratch1, data_addr);
1478       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1479       __ strb(rscratch1, data_addr);
1480       __ b(done);
1481       __ bind(not_null);
1482     } else {
1483       __ cbz(value, done);
1484     }
1485 
1486     add_debug_info_for_null_check_here(op->info_for_exception());
1487     __ load_klass(k_RInfo, array);
1488     __ load_klass(klass_RInfo, value);
1489 
1490     // get instance klass (it's already uncompressed)
1491     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1492     // perform the fast part of the checking logic
1493     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1494     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1495     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1496     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1497     __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1498     // result is a boolean
1499     __ cbzw(k_RInfo, *failure_target);
1500     // fall through to the success case
1501 
1502     if (should_profile) {
1503       Register mdo  = klass_RInfo, recv = k_RInfo;
1504       __ bind(profile_cast_success);
1505       __ mov_metadata(mdo, md->constant_encoding());
1506       __ load_klass(recv, value);
1507       Label update_done;
1508       type_profile_helper(mdo, md, data, recv, &done);
1509       __ b(done);
1510 
1511       __ bind(profile_cast_failure);
1512       __ mov_metadata(mdo, md->constant_encoding());
1513       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1514       __ lea(rscratch2, counter_addr);
1515       __ ldr(rscratch1, Address(rscratch2));
1516       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1517       __ str(rscratch1, Address(rscratch2));
1518       __ b(*stub->entry());
1519     }
1520 
1521     __ bind(done);
1522   } else if (code == lir_checkcast) {
1523     Register obj = op->object()->as_register();
1524     Register dst = op->result_opr()->as_register();
1525     Label success;
1526     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1527     __ bind(success);
1528     if (dst != obj) {
1529       __ mov(dst, obj);
1530     }
1531   } else if (code == lir_instanceof) {
1532     Register obj = op->object()->as_register();
1533     Register dst = op->result_opr()->as_register();
1534     Label success, failure, done;
1535     emit_typecheck_helper(op, &success, &failure, &failure);
1536     __ bind(failure);
1537     __ mov(dst, zr);
1538     __ b(done);
1539     __ bind(success);
1540     __ mov(dst, 1);
1541     __ bind(done);
1542   } else {
1543     ShouldNotReachHere();
1544   }
1545 }
1546 
1547 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1548   __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1549   __ cset(rscratch1, Assembler::NE);
1550   __ membar(__ AnyAny);
1551 }
1552 
1553 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1554   __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1555   __ cset(rscratch1, Assembler::NE);
1556   __ membar(__ AnyAny);
1557 }
1558 
1559 
1560 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1561   assert(VM_Version::supports_cx8(), "wrong machine");
1562   Register addr;
1563   if (op->addr()->is_register()) {
1564     addr = as_reg(op->addr());
1565   } else {
1566     assert(op->addr()->is_address(), "what else?");
1567     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1568     assert(addr_ptr->disp() == 0, "need 0 disp");
1569     assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index");
1570     addr = as_reg(addr_ptr->base());
1571   }
1572   Register newval = as_reg(op->new_value());
1573   Register cmpval = as_reg(op->cmp_value());
1574   Label succeed, fail, around;
1575 
1576   if (op->code() == lir_cas_obj) {
1577     if (UseCompressedOops) {
1578       Register t1 = op->tmp1()->as_register();
1579       assert(op->tmp1()->is_valid(), "must be");
1580       __ encode_heap_oop(t1, cmpval);
1581       cmpval = t1;
1582       __ encode_heap_oop(rscratch2, newval);
1583       newval = rscratch2;
1584       casw(addr, newval, cmpval);
1585     } else {
1586       casl(addr, newval, cmpval);
1587     }
1588   } else if (op->code() == lir_cas_int) {
1589     casw(addr, newval, cmpval);
1590   } else {
1591     casl(addr, newval, cmpval);
1592   }
1593 }
1594 
1595 
1596 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1597 
1598   Assembler::Condition acond, ncond;
1599   switch (condition) {
1600   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1601   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1602   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1603   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1604   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1605   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1606   case lir_cond_belowEqual:
1607   case lir_cond_aboveEqual:
1608   default:                    ShouldNotReachHere();
1609     acond = Assembler::EQ; ncond = Assembler::NE;  // unreachable
1610   }
1611 
1612   assert(result->is_single_cpu() || result->is_double_cpu(),
1613          "expect single register for result");
1614   if (opr1->is_constant() && opr2->is_constant()
1615       && opr1->type() == T_INT && opr2->type() == T_INT) {
1616     jint val1 = opr1->as_jint();
1617     jint val2 = opr2->as_jint();
1618     if (val1 == 0 && val2 == 1) {
1619       __ cset(result->as_register(), ncond);
1620       return;
1621     } else if (val1 == 1 && val2 == 0) {
1622       __ cset(result->as_register(), acond);
1623       return;
1624     }
1625   }
1626 
1627   if (opr1->is_constant() && opr2->is_constant()
1628       && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1629     jlong val1 = opr1->as_jlong();
1630     jlong val2 = opr2->as_jlong();
1631     if (val1 == 0 && val2 == 1) {
1632       __ cset(result->as_register_lo(), ncond);
1633       return;
1634     } else if (val1 == 1 && val2 == 0) {
1635       __ cset(result->as_register_lo(), acond);
1636       return;
1637     }
1638   }
1639 
1640   if (opr1->is_stack()) {
1641     stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1642     opr1 = FrameMap::rscratch1_opr;
1643   } else if (opr1->is_constant()) {
1644     LIR_Opr tmp
1645       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1646     const2reg(opr1, tmp, lir_patch_none, NULL);
1647     opr1 = tmp;
1648   }
1649 
1650   if (opr2->is_stack()) {
1651     stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1652     opr2 = FrameMap::rscratch2_opr;
1653   } else if (opr2->is_constant()) {
1654     LIR_Opr tmp
1655       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1656     const2reg(opr2, tmp, lir_patch_none, NULL);
1657     opr2 = tmp;
1658   }
1659 
1660   if (result->type() == T_LONG)
1661     __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1662   else
1663     __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1664 }
1665 
1666 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1667   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1668 
1669   if (left->is_single_cpu()) {
1670     Register lreg = left->as_register();
1671     Register dreg = as_reg(dest);
1672 
1673     if (right->is_single_cpu()) {
1674       // cpu register - cpu register
1675 
1676       assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1677              "should be");
1678       Register rreg = right->as_register();
1679       switch (code) {
1680       case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1681       case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1682       case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1683       default:      ShouldNotReachHere();
1684       }
1685 
1686     } else if (right->is_double_cpu()) {
1687       Register rreg = right->as_register_lo();
1688       // single_cpu + double_cpu: can happen with obj+long
1689       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1690       switch (code) {
1691       case lir_add: __ add(dreg, lreg, rreg); break;
1692       case lir_sub: __ sub(dreg, lreg, rreg); break;
1693       default: ShouldNotReachHere();
1694       }
1695     } else if (right->is_constant()) {
1696       // cpu register - constant
1697       jlong c;
1698 
1699       // FIXME.  This is fugly: we really need to factor all this logic.
1700       switch(right->type()) {
1701       case T_LONG:
1702         c = right->as_constant_ptr()->as_jlong();
1703         break;
1704       case T_INT:
1705       case T_ADDRESS:
1706         c = right->as_constant_ptr()->as_jint();
1707         break;
1708       default:
1709         ShouldNotReachHere();
1710         c = 0;  // unreachable
1711         break;
1712       }
1713 
1714       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1715       if (c == 0 && dreg == lreg) {
1716         COMMENT("effective nop elided");
1717         return;
1718       }
1719       switch(left->type()) {
1720       case T_INT:
1721         switch (code) {
1722         case lir_add: __ addw(dreg, lreg, c); break;
1723         case lir_sub: __ subw(dreg, lreg, c); break;
1724         default: ShouldNotReachHere();
1725         }
1726         break;
1727       case T_OBJECT:
1728       case T_ADDRESS:
1729         switch (code) {
1730         case lir_add: __ add(dreg, lreg, c); break;
1731         case lir_sub: __ sub(dreg, lreg, c); break;
1732         default: ShouldNotReachHere();
1733         }
1734         break;
1735         ShouldNotReachHere();
1736       }
1737     } else {
1738       ShouldNotReachHere();
1739     }
1740 
1741   } else if (left->is_double_cpu()) {
1742     Register lreg_lo = left->as_register_lo();
1743 
1744     if (right->is_double_cpu()) {
1745       // cpu register - cpu register
1746       Register rreg_lo = right->as_register_lo();
1747       switch (code) {
1748       case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1749       case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1750       case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1751       case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1752       case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1753       default:
1754         ShouldNotReachHere();
1755       }
1756 
1757     } else if (right->is_constant()) {
1758       jlong c = right->as_constant_ptr()->as_jlong_bits();
1759       Register dreg = as_reg(dest);
1760       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1761       if (c == 0 && dreg == lreg_lo) {
1762         COMMENT("effective nop elided");
1763         return;
1764       }
1765       switch (code) {
1766         case lir_add: __ add(dreg, lreg_lo, c); break;
1767         case lir_sub: __ sub(dreg, lreg_lo, c); break;
1768         default:
1769           ShouldNotReachHere();
1770       }
1771     } else {
1772       ShouldNotReachHere();
1773     }
1774   } else if (left->is_single_fpu()) {
1775     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1776     switch (code) {
1777     case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1778     case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1779     case lir_mul_strictfp: // fall through
1780     case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1781     case lir_div_strictfp: // fall through
1782     case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1783     default:
1784       ShouldNotReachHere();
1785     }
1786   } else if (left->is_double_fpu()) {
1787     if (right->is_double_fpu()) {
1788       // fpu register - fpu register
1789       switch (code) {
1790       case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1791       case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1792       case lir_mul_strictfp: // fall through
1793       case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1794       case lir_div_strictfp: // fall through
1795       case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1796       default:
1797         ShouldNotReachHere();
1798       }
1799     } else {
1800       if (right->is_constant()) {
1801         ShouldNotReachHere();
1802       }
1803       ShouldNotReachHere();
1804     }
1805   } else if (left->is_single_stack() || left->is_address()) {
1806     assert(left == dest, "left and dest must be equal");
1807     ShouldNotReachHere();
1808   } else {
1809     ShouldNotReachHere();
1810   }
1811 }
1812 
1813 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1814 
1815 
1816 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1817   switch(code) {
1818   case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1819   case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1820   default      : ShouldNotReachHere();
1821   }
1822 }
1823 
1824 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1825 
1826   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1827   Register Rleft = left->is_single_cpu() ? left->as_register() :
1828                                            left->as_register_lo();
1829    if (dst->is_single_cpu()) {
1830      Register Rdst = dst->as_register();
1831      if (right->is_constant()) {
1832        switch (code) {
1833          case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1834          case lir_logic_or:  __ orrw (Rdst, Rleft, right->as_jint()); break;
1835          case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1836          default: ShouldNotReachHere(); break;
1837        }
1838      } else {
1839        Register Rright = right->is_single_cpu() ? right->as_register() :
1840                                                   right->as_register_lo();
1841        switch (code) {
1842          case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1843          case lir_logic_or:  __ orrw (Rdst, Rleft, Rright); break;
1844          case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1845          default: ShouldNotReachHere(); break;
1846        }
1847      }
1848    } else {
1849      Register Rdst = dst->as_register_lo();
1850      if (right->is_constant()) {
1851        switch (code) {
1852          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1853          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jlong()); break;
1854          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1855          default: ShouldNotReachHere(); break;
1856        }
1857      } else {
1858        Register Rright = right->is_single_cpu() ? right->as_register() :
1859                                                   right->as_register_lo();
1860        switch (code) {
1861          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1862          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
1863          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1864          default: ShouldNotReachHere(); break;
1865        }
1866      }
1867    }
1868 }
1869 
1870 
1871 
1872 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1873 
1874 
1875 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1876   if (opr1->is_constant() && opr2->is_single_cpu()) {
1877     // tableswitch
1878     Register reg = as_reg(opr2);
1879     struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1880     __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1881   } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1882     Register reg1 = as_reg(opr1);
1883     if (opr2->is_single_cpu()) {
1884       // cpu register - cpu register
1885       Register reg2 = opr2->as_register();
1886       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1887         __ cmpoop(reg1, reg2);
1888       } else {
1889         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1890         __ cmpw(reg1, reg2);
1891       }
1892       return;
1893     }
1894     if (opr2->is_double_cpu()) {
1895       // cpu register - cpu register
1896       Register reg2 = opr2->as_register_lo();
1897       __ cmp(reg1, reg2);
1898       return;
1899     }
1900 
1901     if (opr2->is_constant()) {
1902       bool is_32bit = false; // width of register operand
1903       jlong imm;
1904 
1905       switch(opr2->type()) {
1906       case T_INT:
1907         imm = opr2->as_constant_ptr()->as_jint();
1908         is_32bit = true;
1909         break;
1910       case T_LONG:
1911         imm = opr2->as_constant_ptr()->as_jlong();
1912         break;
1913       case T_ADDRESS:
1914         imm = opr2->as_constant_ptr()->as_jint();
1915         break;
1916       case T_METADATA:
1917         imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
1918         break;
1919       case T_OBJECT:
1920       case T_ARRAY:
1921         jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
1922         __ cmpoop(reg1, rscratch1);
1923         return;
1924       default:
1925         ShouldNotReachHere();
1926         imm = 0;  // unreachable
1927         break;
1928       }
1929 
1930       if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1931         if (is_32bit)
1932           __ cmpw(reg1, imm);
1933         else
1934           __ cmp(reg1, imm);
1935         return;
1936       } else {
1937         __ mov(rscratch1, imm);
1938         if (is_32bit)
1939           __ cmpw(reg1, rscratch1);
1940         else
1941           __ cmp(reg1, rscratch1);
1942         return;
1943       }
1944     } else
1945       ShouldNotReachHere();
1946   } else if (opr1->is_single_fpu()) {
1947     FloatRegister reg1 = opr1->as_float_reg();
1948     assert(opr2->is_single_fpu(), "expect single float register");
1949     FloatRegister reg2 = opr2->as_float_reg();
1950     __ fcmps(reg1, reg2);
1951   } else if (opr1->is_double_fpu()) {
1952     FloatRegister reg1 = opr1->as_double_reg();
1953     assert(opr2->is_double_fpu(), "expect double float register");
1954     FloatRegister reg2 = opr2->as_double_reg();
1955     __ fcmpd(reg1, reg2);
1956   } else {
1957     ShouldNotReachHere();
1958   }
1959 }
1960 
1961 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1962   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1963     bool is_unordered_less = (code == lir_ucmp_fd2i);
1964     if (left->is_single_fpu()) {
1965       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1966     } else if (left->is_double_fpu()) {
1967       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1968     } else {
1969       ShouldNotReachHere();
1970     }
1971   } else if (code == lir_cmp_l2i) {
1972     Label done;
1973     __ cmp(left->as_register_lo(), right->as_register_lo());
1974     __ mov(dst->as_register(), (u_int64_t)-1L);
1975     __ br(Assembler::LT, done);
1976     __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1977     __ bind(done);
1978   } else {
1979     ShouldNotReachHere();
1980   }
1981 }
1982 
1983 
1984 void LIR_Assembler::align_call(LIR_Code code) {  }
1985 
1986 
1987 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1988   address call = __ trampoline_call(Address(op->addr(), rtype));
1989   if (call == NULL) {
1990     bailout("trampoline stub overflow");
1991     return;
1992   }
1993   add_call_info(code_offset(), op->info());
1994 }
1995 
1996 
1997 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1998   address call = __ ic_call(op->addr());
1999   if (call == NULL) {
2000     bailout("trampoline stub overflow");
2001     return;
2002   }
2003   add_call_info(code_offset(), op->info());
2004 }
2005 
2006 
2007 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2008 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2009   ShouldNotReachHere();
2010 }
2011 
2012 
2013 void LIR_Assembler::emit_static_call_stub() {
2014   address call_pc = __ pc();
2015   address stub = __ start_a_stub(call_stub_size());
2016   if (stub == NULL) {
2017     bailout("static call stub overflow");
2018     return;
2019   }
2020 
2021   int start = __ offset();
2022 
2023   __ relocate(static_stub_Relocation::spec(call_pc));
2024   __ emit_static_call_stub();
2025 
2026   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
2027         <= call_stub_size(), "stub too big");
2028   __ end_a_stub();
2029 }
2030 
2031 
2032 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2033   assert(exceptionOop->as_register() == r0, "must match");
2034   assert(exceptionPC->as_register() == r3, "must match");
2035 
2036   // exception object is not added to oop map by LinearScan
2037   // (LinearScan assumes that no oops are in fixed registers)
2038   info->add_register_oop(exceptionOop);
2039   Runtime1::StubID unwind_id;
2040 
2041   // get current pc information
2042   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2043   int pc_for_athrow_offset = __ offset();
2044   InternalAddress pc_for_athrow(__ pc());
2045   __ adr(exceptionPC->as_register(), pc_for_athrow);
2046   add_call_info(pc_for_athrow_offset, info); // for exception handler
2047 
2048   __ verify_not_null_oop(r0);
2049   // search an exception handler (r0: exception oop, r3: throwing pc)
2050   if (compilation()->has_fpu_code()) {
2051     unwind_id = Runtime1::handle_exception_id;
2052   } else {
2053     unwind_id = Runtime1::handle_exception_nofpu_id;
2054   }
2055   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2056 
2057   // FIXME: enough room for two byte trap   ????
2058   __ nop();
2059 }
2060 
2061 
2062 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2063   assert(exceptionOop->as_register() == r0, "must match");
2064 
2065   __ b(_unwind_handler_entry);
2066 }
2067 
2068 
2069 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2070   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2071   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2072 
2073   switch (left->type()) {
2074     case T_INT: {
2075       switch (code) {
2076       case lir_shl:  __ lslvw (dreg, lreg, count->as_register()); break;
2077       case lir_shr:  __ asrvw (dreg, lreg, count->as_register()); break;
2078       case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2079       default:
2080         ShouldNotReachHere();
2081         break;
2082       }
2083       break;
2084     case T_LONG:
2085     case T_ADDRESS:
2086     case T_OBJECT:
2087       switch (code) {
2088       case lir_shl:  __ lslv (dreg, lreg, count->as_register()); break;
2089       case lir_shr:  __ asrv (dreg, lreg, count->as_register()); break;
2090       case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2091       default:
2092         ShouldNotReachHere();
2093         break;
2094       }
2095       break;
2096     default:
2097       ShouldNotReachHere();
2098       break;
2099     }
2100   }
2101 }
2102 
2103 
2104 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2105   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2106   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2107 
2108   switch (left->type()) {
2109     case T_INT: {
2110       switch (code) {
2111       case lir_shl:  __ lslw (dreg, lreg, count); break;
2112       case lir_shr:  __ asrw (dreg, lreg, count); break;
2113       case lir_ushr: __ lsrw (dreg, lreg, count); break;
2114       default:
2115         ShouldNotReachHere();
2116         break;
2117       }
2118       break;
2119     case T_LONG:
2120     case T_ADDRESS:
2121     case T_OBJECT:
2122       switch (code) {
2123       case lir_shl:  __ lsl (dreg, lreg, count); break;
2124       case lir_shr:  __ asr (dreg, lreg, count); break;
2125       case lir_ushr: __ lsr (dreg, lreg, count); break;
2126       default:
2127         ShouldNotReachHere();
2128         break;
2129       }
2130       break;
2131     default:
2132       ShouldNotReachHere();
2133       break;
2134     }
2135   }
2136 }
2137 
2138 
2139 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2140   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2141   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2142   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2143   __ str (r, Address(sp, offset_from_rsp_in_bytes));
2144 }
2145 
2146 
2147 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2148   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2149   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2150   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2151   __ mov (rscratch1, c);
2152   __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2153 }
2154 
2155 
2156 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2157   ShouldNotReachHere();
2158   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2159   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2160   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2161   __ lea(rscratch1, __ constant_oop_address(o));
2162   __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2163 }
2164 
2165 
2166 // This code replaces a call to arraycopy; no exception may
2167 // be thrown in this code, they must be thrown in the System.arraycopy
2168 // activation frame; we could save some checks if this would not be the case
2169 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2170   ciArrayKlass* default_type = op->expected_type();
2171   Register src = op->src()->as_register();
2172   Register dst = op->dst()->as_register();
2173   Register src_pos = op->src_pos()->as_register();
2174   Register dst_pos = op->dst_pos()->as_register();
2175   Register length  = op->length()->as_register();
2176   Register tmp = op->tmp()->as_register();
2177 
2178   CodeStub* stub = op->stub();
2179   int flags = op->flags();
2180   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2181   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2182 
2183   // if we don't know anything, just go through the generic arraycopy
2184   if (default_type == NULL // || basic_type == T_OBJECT
2185       ) {
2186     Label done;
2187     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2188 
2189     // Save the arguments in case the generic arraycopy fails and we
2190     // have to fall back to the JNI stub
2191     __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2192     __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2193     __ str(src,              Address(sp, 4*BytesPerWord));
2194 
2195     address copyfunc_addr = StubRoutines::generic_arraycopy();
2196     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2197 
2198     // The arguments are in java calling convention so we shift them
2199     // to C convention
2200     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2201     __ mov(c_rarg0, j_rarg0);
2202     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2203     __ mov(c_rarg1, j_rarg1);
2204     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2205     __ mov(c_rarg2, j_rarg2);
2206     assert_different_registers(c_rarg3, j_rarg4);
2207     __ mov(c_rarg3, j_rarg3);
2208     __ mov(c_rarg4, j_rarg4);
2209 #ifndef PRODUCT
2210     if (PrintC1Statistics) {
2211       __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2212     }
2213 #endif
2214     __ far_call(RuntimeAddress(copyfunc_addr));
2215 
2216     __ cbz(r0, *stub->continuation());
2217 
2218     // Reload values from the stack so they are where the stub
2219     // expects them.
2220     __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2221     __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2222     __ ldr(src,              Address(sp, 4*BytesPerWord));
2223 
2224     // r0 is -1^K where K == partial copied count
2225     __ eonw(rscratch1, r0, zr);
2226     // adjust length down and src/end pos up by partial copied count
2227     __ subw(length, length, rscratch1);
2228     __ addw(src_pos, src_pos, rscratch1);
2229     __ addw(dst_pos, dst_pos, rscratch1);
2230     __ b(*stub->entry());
2231 
2232     __ bind(*stub->continuation());
2233     return;
2234   }
2235 
2236   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2237 
2238   int elem_size = type2aelembytes(basic_type);
2239   int shift_amount;
2240   int scale = exact_log2(elem_size);
2241 
2242   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2243   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2244   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2245   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2246 
2247   // test for NULL
2248   if (flags & LIR_OpArrayCopy::src_null_check) {
2249     __ cbz(src, *stub->entry());
2250   }
2251   if (flags & LIR_OpArrayCopy::dst_null_check) {
2252     __ cbz(dst, *stub->entry());
2253   }
2254 
2255   // If the compiler was not able to prove that exact type of the source or the destination
2256   // of the arraycopy is an array type, check at runtime if the source or the destination is
2257   // an instance type.
2258   if (flags & LIR_OpArrayCopy::type_check) {
2259     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2260       __ load_klass(tmp, dst);
2261       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2262       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2263       __ br(Assembler::GE, *stub->entry());
2264     }
2265 
2266     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2267       __ load_klass(tmp, src);
2268       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2269       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2270       __ br(Assembler::GE, *stub->entry());
2271     }
2272   }
2273 
2274   // check if negative
2275   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2276     __ cmpw(src_pos, 0);
2277     __ br(Assembler::LT, *stub->entry());
2278   }
2279   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2280     __ cmpw(dst_pos, 0);
2281     __ br(Assembler::LT, *stub->entry());
2282   }
2283 
2284   if (flags & LIR_OpArrayCopy::length_positive_check) {
2285     __ cmpw(length, 0);
2286     __ br(Assembler::LT, *stub->entry());
2287   }
2288 
2289   if (flags & LIR_OpArrayCopy::src_range_check) {
2290     __ addw(tmp, src_pos, length);
2291     __ ldrw(rscratch1, src_length_addr);
2292     __ cmpw(tmp, rscratch1);
2293     __ br(Assembler::HI, *stub->entry());
2294   }
2295   if (flags & LIR_OpArrayCopy::dst_range_check) {
2296     __ addw(tmp, dst_pos, length);
2297     __ ldrw(rscratch1, dst_length_addr);
2298     __ cmpw(tmp, rscratch1);
2299     __ br(Assembler::HI, *stub->entry());
2300   }
2301 
2302   if (flags & LIR_OpArrayCopy::type_check) {
2303     // We don't know the array types are compatible
2304     if (basic_type != T_OBJECT) {
2305       // Simple test for basic type arrays
2306       if (UseCompressedClassPointers) {
2307         __ ldrw(tmp, src_klass_addr);
2308         __ ldrw(rscratch1, dst_klass_addr);
2309         __ cmpw(tmp, rscratch1);
2310       } else {
2311         __ ldr(tmp, src_klass_addr);
2312         __ ldr(rscratch1, dst_klass_addr);
2313         __ cmp(tmp, rscratch1);
2314       }
2315       __ br(Assembler::NE, *stub->entry());
2316     } else {
2317       // For object arrays, if src is a sub class of dst then we can
2318       // safely do the copy.
2319       Label cont, slow;
2320 
2321 #define PUSH(r1, r2)                                    \
2322       stp(r1, r2, __ pre(sp, -2 * wordSize));
2323 
2324 #define POP(r1, r2)                                     \
2325       ldp(r1, r2, __ post(sp, 2 * wordSize));
2326 
2327       __ PUSH(src, dst);
2328 
2329       __ load_klass(src, src);
2330       __ load_klass(dst, dst);
2331 
2332       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2333 
2334       __ PUSH(src, dst);
2335       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2336       __ POP(src, dst);
2337 
2338       __ cbnz(src, cont);
2339 
2340       __ bind(slow);
2341       __ POP(src, dst);
2342 
2343       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2344       if (copyfunc_addr != NULL) { // use stub if available
2345         // src is not a sub class of dst so we have to do a
2346         // per-element check.
2347 
2348         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2349         if ((flags & mask) != mask) {
2350           // Check that at least both of them object arrays.
2351           assert(flags & mask, "one of the two should be known to be an object array");
2352 
2353           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2354             __ load_klass(tmp, src);
2355           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2356             __ load_klass(tmp, dst);
2357           }
2358           int lh_offset = in_bytes(Klass::layout_helper_offset());
2359           Address klass_lh_addr(tmp, lh_offset);
2360           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2361           __ ldrw(rscratch1, klass_lh_addr);
2362           __ mov(rscratch2, objArray_lh);
2363           __ eorw(rscratch1, rscratch1, rscratch2);
2364           __ cbnzw(rscratch1, *stub->entry());
2365         }
2366 
2367        // Spill because stubs can use any register they like and it's
2368        // easier to restore just those that we care about.
2369         __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2370         __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2371         __ str(src,              Address(sp, 4*BytesPerWord));
2372 
2373         __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2374         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2375         assert_different_registers(c_rarg0, dst, dst_pos, length);
2376         __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2377         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2378         assert_different_registers(c_rarg1, dst, length);
2379         __ uxtw(c_rarg2, length);
2380         assert_different_registers(c_rarg2, dst);
2381 
2382         __ load_klass(c_rarg4, dst);
2383         __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2384         __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2385         __ far_call(RuntimeAddress(copyfunc_addr));
2386 
2387 #ifndef PRODUCT
2388         if (PrintC1Statistics) {
2389           Label failed;
2390           __ cbnz(r0, failed);
2391           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2392           __ bind(failed);
2393         }
2394 #endif
2395 
2396         __ cbz(r0, *stub->continuation());
2397 
2398 #ifndef PRODUCT
2399         if (PrintC1Statistics) {
2400           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2401         }
2402 #endif
2403         assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2404 
2405         // Restore previously spilled arguments
2406         __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2407         __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2408         __ ldr(src,              Address(sp, 4*BytesPerWord));
2409 
2410         // return value is -1^K where K is partial copied count
2411         __ eonw(rscratch1, r0, zr);
2412         // adjust length down and src/end pos up by partial copied count
2413         __ subw(length, length, rscratch1);
2414         __ addw(src_pos, src_pos, rscratch1);
2415         __ addw(dst_pos, dst_pos, rscratch1);
2416       }
2417 
2418       __ b(*stub->entry());
2419 
2420       __ bind(cont);
2421       __ POP(src, dst);
2422     }
2423   }
2424 
2425 #ifdef ASSERT
2426   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2427     // Sanity check the known type with the incoming class.  For the
2428     // primitive case the types must match exactly with src.klass and
2429     // dst.klass each exactly matching the default type.  For the
2430     // object array case, if no type check is needed then either the
2431     // dst type is exactly the expected type and the src type is a
2432     // subtype which we can't check or src is the same array as dst
2433     // but not necessarily exactly of type default_type.
2434     Label known_ok, halt;
2435     __ mov_metadata(tmp, default_type->constant_encoding());
2436     if (UseCompressedClassPointers) {
2437       __ encode_klass_not_null(tmp);
2438     }
2439 
2440     if (basic_type != T_OBJECT) {
2441 
2442       if (UseCompressedClassPointers) {
2443         __ ldrw(rscratch1, dst_klass_addr);
2444         __ cmpw(tmp, rscratch1);
2445       } else {
2446         __ ldr(rscratch1, dst_klass_addr);
2447         __ cmp(tmp, rscratch1);
2448       }
2449       __ br(Assembler::NE, halt);
2450       if (UseCompressedClassPointers) {
2451         __ ldrw(rscratch1, src_klass_addr);
2452         __ cmpw(tmp, rscratch1);
2453       } else {
2454         __ ldr(rscratch1, src_klass_addr);
2455         __ cmp(tmp, rscratch1);
2456       }
2457       __ br(Assembler::EQ, known_ok);
2458     } else {
2459       if (UseCompressedClassPointers) {
2460         __ ldrw(rscratch1, dst_klass_addr);
2461         __ cmpw(tmp, rscratch1);
2462       } else {
2463         __ ldr(rscratch1, dst_klass_addr);
2464         __ cmp(tmp, rscratch1);
2465       }
2466       __ br(Assembler::EQ, known_ok);
2467       __ cmp(src, dst);
2468       __ br(Assembler::EQ, known_ok);
2469     }
2470     __ bind(halt);
2471     __ stop("incorrect type information in arraycopy");
2472     __ bind(known_ok);
2473   }
2474 #endif
2475 
2476 #ifndef PRODUCT
2477   if (PrintC1Statistics) {
2478     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2479   }
2480 #endif
2481 
2482   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2483   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2484   assert_different_registers(c_rarg0, dst, dst_pos, length);
2485   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2486   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2487   assert_different_registers(c_rarg1, dst, length);
2488   __ uxtw(c_rarg2, length);
2489   assert_different_registers(c_rarg2, dst);
2490 
2491   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2492   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2493   const char *name;
2494   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2495 
2496  CodeBlob *cb = CodeCache::find_blob(entry);
2497  if (cb) {
2498    __ far_call(RuntimeAddress(entry));
2499  } else {
2500    __ call_VM_leaf(entry, 3);
2501  }
2502 
2503   __ bind(*stub->continuation());
2504 }
2505 
2506 
2507 
2508 
2509 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2510   Register obj = op->obj_opr()->as_register();  // may not be an oop
2511   Register hdr = op->hdr_opr()->as_register();
2512   Register lock = op->lock_opr()->as_register();
2513   if (!UseFastLocking) {
2514     __ b(*op->stub()->entry());
2515   } else if (op->code() == lir_lock) {
2516     Register scratch = noreg;
2517     if (UseBiasedLocking) {
2518       scratch = op->scratch_opr()->as_register();
2519     }
2520     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2521     // add debug info for NullPointerException only if one is possible
2522     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2523     if (op->info() != NULL) {
2524       add_debug_info_for_null_check(null_check_offset, op->info());
2525     }
2526     // done
2527   } else if (op->code() == lir_unlock) {
2528     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2529     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2530   } else {
2531     Unimplemented();
2532   }
2533   __ bind(*op->stub()->continuation());
2534 }
2535 
2536 
2537 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2538   ciMethod* method = op->profiled_method();
2539   int bci          = op->profiled_bci();
2540   ciMethod* callee = op->profiled_callee();
2541 
2542   // Update counter for all call types
2543   ciMethodData* md = method->method_data_or_null();
2544   assert(md != NULL, "Sanity");
2545   ciProfileData* data = md->bci_to_data(bci);
2546   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2547   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2548   Register mdo  = op->mdo()->as_register();
2549   __ mov_metadata(mdo, md->constant_encoding());
2550   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2551   // Perform additional virtual call profiling for invokevirtual and
2552   // invokeinterface bytecodes
2553   if (op->should_profile_receiver_type()) {
2554     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2555     Register recv = op->recv()->as_register();
2556     assert_different_registers(mdo, recv);
2557     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2558     ciKlass* known_klass = op->known_holder();
2559     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2560       // We know the type that will be seen at this call site; we can
2561       // statically update the MethodData* rather than needing to do
2562       // dynamic tests on the receiver type
2563 
2564       // NOTE: we should probably put a lock around this search to
2565       // avoid collisions by concurrent compilations
2566       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2567       uint i;
2568       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2569         ciKlass* receiver = vc_data->receiver(i);
2570         if (known_klass->equals(receiver)) {
2571           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2572           __ addptr(data_addr, DataLayout::counter_increment);
2573           return;
2574         }
2575       }
2576 
2577       // Receiver type not found in profile data; select an empty slot
2578 
2579       // Note that this is less efficient than it should be because it
2580       // always does a write to the receiver part of the
2581       // VirtualCallData rather than just the first time
2582       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2583         ciKlass* receiver = vc_data->receiver(i);
2584         if (receiver == NULL) {
2585           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2586           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2587           __ lea(rscratch2, recv_addr);
2588           __ str(rscratch1, Address(rscratch2));
2589           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2590           __ addptr(data_addr, DataLayout::counter_increment);
2591           return;
2592         }
2593       }
2594     } else {
2595       __ load_klass(recv, recv);
2596       Label update_done;
2597       type_profile_helper(mdo, md, data, recv, &update_done);
2598       // Receiver did not match any saved receiver and there is no empty row for it.
2599       // Increment total counter to indicate polymorphic case.
2600       __ addptr(counter_addr, DataLayout::counter_increment);
2601 
2602       __ bind(update_done);
2603     }
2604   } else {
2605     // Static call
2606     __ addptr(counter_addr, DataLayout::counter_increment);
2607   }
2608 }
2609 
2610 
2611 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2612   Unimplemented();
2613 }
2614 
2615 
2616 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2617   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2618 }
2619 
2620 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2621   assert(op->crc()->is_single_cpu(),  "crc must be register");
2622   assert(op->val()->is_single_cpu(),  "byte value must be register");
2623   assert(op->result_opr()->is_single_cpu(), "result must be register");
2624   Register crc = op->crc()->as_register();
2625   Register val = op->val()->as_register();
2626   Register res = op->result_opr()->as_register();
2627 
2628   assert_different_registers(val, crc, res);
2629   unsigned long offset;
2630   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2631   if (offset) __ add(res, res, offset);
2632 
2633   __ mvnw(crc, crc); // ~crc
2634   __ update_byte_crc32(crc, val, res);
2635   __ mvnw(res, crc); // ~crc
2636 }
2637 
2638 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2639   COMMENT("emit_profile_type {");
2640   Register obj = op->obj()->as_register();
2641   Register tmp = op->tmp()->as_pointer_register();
2642   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2643   ciKlass* exact_klass = op->exact_klass();
2644   intptr_t current_klass = op->current_klass();
2645   bool not_null = op->not_null();
2646   bool no_conflict = op->no_conflict();
2647 
2648   Label update, next, none;
2649 
2650   bool do_null = !not_null;
2651   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2652   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2653 
2654   assert(do_null || do_update, "why are we here?");
2655   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2656   assert(mdo_addr.base() != rscratch1, "wrong register");
2657 
2658   __ verify_oop(obj);
2659 
2660   if (tmp != obj) {
2661     __ mov(tmp, obj);
2662   }
2663   if (do_null) {
2664     __ cbnz(tmp, update);
2665     if (!TypeEntries::was_null_seen(current_klass)) {
2666       __ ldr(rscratch2, mdo_addr);
2667       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2668       __ str(rscratch2, mdo_addr);
2669     }
2670     if (do_update) {
2671 #ifndef ASSERT
2672       __ b(next);
2673     }
2674 #else
2675       __ b(next);
2676     }
2677   } else {
2678     __ cbnz(tmp, update);
2679     __ stop("unexpected null obj");
2680 #endif
2681   }
2682 
2683   __ bind(update);
2684 
2685   if (do_update) {
2686 #ifdef ASSERT
2687     if (exact_klass != NULL) {
2688       Label ok;
2689       __ load_klass(tmp, tmp);
2690       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2691       __ eor(rscratch1, tmp, rscratch1);
2692       __ cbz(rscratch1, ok);
2693       __ stop("exact klass and actual klass differ");
2694       __ bind(ok);
2695     }
2696 #endif
2697     if (!no_conflict) {
2698       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2699         if (exact_klass != NULL) {
2700           __ mov_metadata(tmp, exact_klass->constant_encoding());
2701         } else {
2702           __ load_klass(tmp, tmp);
2703         }
2704 
2705         __ ldr(rscratch2, mdo_addr);
2706         __ eor(tmp, tmp, rscratch2);
2707         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2708         // klass seen before, nothing to do. The unknown bit may have been
2709         // set already but no need to check.
2710         __ cbz(rscratch1, next);
2711 
2712         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2713 
2714         if (TypeEntries::is_type_none(current_klass)) {
2715           __ cbz(rscratch2, none);
2716           __ cmp(rscratch2, TypeEntries::null_seen);
2717           __ br(Assembler::EQ, none);
2718           // There is a chance that the checks above (re-reading profiling
2719           // data from memory) fail if another thread has just set the
2720           // profiling to this obj's klass
2721           __ dmb(Assembler::ISHLD);
2722           __ ldr(rscratch2, mdo_addr);
2723           __ eor(tmp, tmp, rscratch2);
2724           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2725           __ cbz(rscratch1, next);
2726         }
2727       } else {
2728         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2729                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2730 
2731         __ ldr(tmp, mdo_addr);
2732         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2733       }
2734 
2735       // different than before. Cannot keep accurate profile.
2736       __ ldr(rscratch2, mdo_addr);
2737       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2738       __ str(rscratch2, mdo_addr);
2739 
2740       if (TypeEntries::is_type_none(current_klass)) {
2741         __ b(next);
2742 
2743         __ bind(none);
2744         // first time here. Set profile type.
2745         __ str(tmp, mdo_addr);
2746       }
2747     } else {
2748       // There's a single possible klass at this profile point
2749       assert(exact_klass != NULL, "should be");
2750       if (TypeEntries::is_type_none(current_klass)) {
2751         __ mov_metadata(tmp, exact_klass->constant_encoding());
2752         __ ldr(rscratch2, mdo_addr);
2753         __ eor(tmp, tmp, rscratch2);
2754         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2755         __ cbz(rscratch1, next);
2756 #ifdef ASSERT
2757         {
2758           Label ok;
2759           __ ldr(rscratch1, mdo_addr);
2760           __ cbz(rscratch1, ok);
2761           __ cmp(rscratch1, TypeEntries::null_seen);
2762           __ br(Assembler::EQ, ok);
2763           // may have been set by another thread
2764           __ dmb(Assembler::ISHLD);
2765           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2766           __ ldr(rscratch2, mdo_addr);
2767           __ eor(rscratch2, rscratch1, rscratch2);
2768           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2769           __ cbz(rscratch2, ok);
2770 
2771           __ stop("unexpected profiling mismatch");
2772           __ bind(ok);
2773         }
2774 #endif
2775         // first time here. Set profile type.
2776         __ ldr(tmp, mdo_addr);
2777       } else {
2778         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2779                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2780 
2781         __ ldr(tmp, mdo_addr);
2782         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2783 
2784         __ orr(tmp, tmp, TypeEntries::type_unknown);
2785         __ str(tmp, mdo_addr);
2786         // FIXME: Write barrier needed here?
2787       }
2788     }
2789 
2790     __ bind(next);
2791   }
2792   COMMENT("} emit_profile_type");
2793 }
2794 
2795 
2796 void LIR_Assembler::align_backward_branch_target() {
2797 }
2798 
2799 
2800 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2801   // tmp must be unused
2802   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2803 
2804   if (left->is_single_cpu()) {
2805     assert(dest->is_single_cpu(), "expect single result reg");
2806     __ negw(dest->as_register(), left->as_register());
2807   } else if (left->is_double_cpu()) {
2808     assert(dest->is_double_cpu(), "expect double result reg");
2809     __ neg(dest->as_register_lo(), left->as_register_lo());
2810   } else if (left->is_single_fpu()) {
2811     assert(dest->is_single_fpu(), "expect single float result reg");
2812     __ fnegs(dest->as_float_reg(), left->as_float_reg());
2813   } else {
2814     assert(left->is_double_fpu(), "expect double float operand reg");
2815     assert(dest->is_double_fpu(), "expect double float result reg");
2816     __ fnegd(dest->as_double_reg(), left->as_double_reg());
2817   }
2818 }
2819 
2820 
2821 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2822 #if INCLUDE_SHENANDOAHGC
2823   if (UseShenandoahGC && patch_code != lir_patch_none) {
2824     deoptimize_trap(info);
2825     return;
2826   }
2827 #endif
2828   assert(patch_code == lir_patch_none, "Patch code not supported");
2829   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2830 }
2831 
2832 
2833 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2834   assert(!tmp->is_valid(), "don't need temporary");
2835 
2836   CodeBlob *cb = CodeCache::find_blob(dest);
2837   if (cb) {
2838     __ far_call(RuntimeAddress(dest));
2839   } else {
2840     __ mov(rscratch1, RuntimeAddress(dest));
2841     __ blr(rscratch1);
2842   }
2843 
2844   if (info != NULL) {
2845     add_call_info_here(info);
2846   }
2847   __ maybe_isb();
2848 }
2849 
2850 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2851   if (dest->is_address() || src->is_address()) {
2852     move_op(src, dest, type, lir_patch_none, info,
2853             /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2854   } else {
2855     ShouldNotReachHere();
2856   }
2857 }
2858 
2859 #ifdef ASSERT
2860 // emit run-time assertion
2861 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2862   assert(op->code() == lir_assert, "must be");
2863 
2864   if (op->in_opr1()->is_valid()) {
2865     assert(op->in_opr2()->is_valid(), "both operands must be valid");
2866     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2867   } else {
2868     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2869     assert(op->condition() == lir_cond_always, "no other conditions allowed");
2870   }
2871 
2872   Label ok;
2873   if (op->condition() != lir_cond_always) {
2874     Assembler::Condition acond = Assembler::AL;
2875     switch (op->condition()) {
2876       case lir_cond_equal:        acond = Assembler::EQ;  break;
2877       case lir_cond_notEqual:     acond = Assembler::NE;  break;
2878       case lir_cond_less:         acond = Assembler::LT;  break;
2879       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
2880       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
2881       case lir_cond_greater:      acond = Assembler::GT;  break;
2882       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
2883       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
2884       default:                    ShouldNotReachHere();
2885     }
2886     __ br(acond, ok);
2887   }
2888   if (op->halt()) {
2889     const char* str = __ code_string(op->msg());
2890     __ stop(str);
2891   } else {
2892     breakpoint();
2893   }
2894   __ bind(ok);
2895 }
2896 #endif
2897 
2898 #ifndef PRODUCT
2899 #define COMMENT(x)   do { __ block_comment(x); } while (0)
2900 #else
2901 #define COMMENT(x)
2902 #endif
2903 
2904 void LIR_Assembler::membar() {
2905   COMMENT("membar");
2906   __ membar(MacroAssembler::AnyAny);
2907 }
2908 
2909 void LIR_Assembler::membar_acquire() {
2910   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2911 }
2912 
2913 void LIR_Assembler::membar_release() {
2914   __ membar(Assembler::LoadStore|Assembler::StoreStore);
2915 }
2916 
2917 void LIR_Assembler::membar_loadload() {
2918   __ membar(Assembler::LoadLoad);
2919 }
2920 
2921 void LIR_Assembler::membar_storestore() {
2922   __ membar(MacroAssembler::StoreStore);
2923 }
2924 
2925 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2926 
2927 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2928 
2929 void LIR_Assembler::on_spin_wait() {
2930   Unimplemented();
2931 }
2932 
2933 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2934   __ mov(result_reg->as_register(), rthread);
2935 }
2936 
2937 
2938 void LIR_Assembler::peephole(LIR_List *lir) {
2939 #if 0
2940   if (tableswitch_count >= max_tableswitches)
2941     return;
2942 
2943   /*
2944     This finite-state automaton recognizes sequences of compare-and-
2945     branch instructions.  We will turn them into a tableswitch.  You
2946     could argue that C1 really shouldn't be doing this sort of
2947     optimization, but without it the code is really horrible.
2948   */
2949 
2950   enum { start_s, cmp1_s, beq_s, cmp_s } state;
2951   int first_key, last_key = -2147483648;
2952   int next_key = 0;
2953   int start_insn = -1;
2954   int last_insn = -1;
2955   Register reg = noreg;
2956   LIR_Opr reg_opr;
2957   state = start_s;
2958 
2959   LIR_OpList* inst = lir->instructions_list();
2960   for (int i = 0; i < inst->length(); i++) {
2961     LIR_Op* op = inst->at(i);
2962     switch (state) {
2963     case start_s:
2964       first_key = -1;
2965       start_insn = i;
2966       switch (op->code()) {
2967       case lir_cmp:
2968         LIR_Opr opr1 = op->as_Op2()->in_opr1();
2969         LIR_Opr opr2 = op->as_Op2()->in_opr2();
2970         if (opr1->is_cpu_register() && opr1->is_single_cpu()
2971             && opr2->is_constant()
2972             && opr2->type() == T_INT) {
2973           reg_opr = opr1;
2974           reg = opr1->as_register();
2975           first_key = opr2->as_constant_ptr()->as_jint();
2976           next_key = first_key + 1;
2977           state = cmp_s;
2978           goto next_state;
2979         }
2980         break;
2981       }
2982       break;
2983     case cmp_s:
2984       switch (op->code()) {
2985       case lir_branch:
2986         if (op->as_OpBranch()->cond() == lir_cond_equal) {
2987           state = beq_s;
2988           last_insn = i;
2989           goto next_state;
2990         }
2991       }
2992       state = start_s;
2993       break;
2994     case beq_s:
2995       switch (op->code()) {
2996       case lir_cmp: {
2997         LIR_Opr opr1 = op->as_Op2()->in_opr1();
2998         LIR_Opr opr2 = op->as_Op2()->in_opr2();
2999         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3000             && opr1->as_register() == reg
3001             && opr2->is_constant()
3002             && opr2->type() == T_INT
3003             && opr2->as_constant_ptr()->as_jint() == next_key) {
3004           last_key = next_key;
3005           next_key++;
3006           state = cmp_s;
3007           goto next_state;
3008         }
3009       }
3010       }
3011       last_key = next_key;
3012       state = start_s;
3013       break;
3014     default:
3015       assert(false, "impossible state");
3016     }
3017     if (state == start_s) {
3018       if (first_key < last_key - 5L && reg != noreg) {
3019         {
3020           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3021           //        reg->encoding(),
3022           //        start_insn, first_key, last_key);
3023           //   for (int i = 0; i < inst->length(); i++) {
3024           //     inst->at(i)->print();
3025           //     tty->print("\n");
3026           //   }
3027           //   tty->print("\n");
3028         }
3029 
3030         struct tableswitch *sw = &switches[tableswitch_count];
3031         sw->_insn_index = start_insn, sw->_first_key = first_key,
3032           sw->_last_key = last_key, sw->_reg = reg;
3033         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3034         {
3035           // Insert the new table of branches
3036           int offset = last_insn;
3037           for (int n = first_key; n < last_key; n++) {
3038             inst->insert_before
3039               (last_insn + 1,
3040                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3041                                 inst->at(offset)->as_OpBranch()->label()));
3042             offset -= 2, i++;
3043           }
3044         }
3045         // Delete all the old compare-and-branch instructions
3046         for (int n = first_key; n < last_key; n++) {
3047           inst->remove_at(start_insn);
3048           inst->remove_at(start_insn);
3049         }
3050         // Insert the tableswitch instruction
3051         inst->insert_before(start_insn,
3052                             new LIR_Op2(lir_cmp, lir_cond_always,
3053                                         LIR_OprFact::intConst(tableswitch_count),
3054                                         reg_opr));
3055         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3056         tableswitch_count++;
3057       }
3058       reg = noreg;
3059       last_key = -2147483648;
3060     }
3061   next_state:
3062     ;
3063   }
3064 #endif
3065 }
3066 
3067 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3068   Address addr = as_Address(src->as_address_ptr());
3069   BasicType type = src->type();
3070   bool is_oop = type == T_OBJECT || type == T_ARRAY;
3071 
3072   void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3073   void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3074 
3075   switch(type) {
3076   case T_INT:
3077     xchg = &MacroAssembler::atomic_xchgalw;
3078     add = &MacroAssembler::atomic_addalw;
3079     break;
3080   case T_LONG:
3081     xchg = &MacroAssembler::atomic_xchgal;
3082     add = &MacroAssembler::atomic_addal;
3083     break;
3084   case T_OBJECT:
3085   case T_ARRAY:
3086     if (UseCompressedOops) {
3087       xchg = &MacroAssembler::atomic_xchgalw;
3088       add = &MacroAssembler::atomic_addalw;
3089     } else {
3090       xchg = &MacroAssembler::atomic_xchgal;
3091       add = &MacroAssembler::atomic_addal;
3092     }
3093     break;
3094   default:
3095     ShouldNotReachHere();
3096     xchg = &MacroAssembler::atomic_xchgal;
3097     add = &MacroAssembler::atomic_addal; // unreachable
3098   }
3099 
3100   switch (code) {
3101   case lir_xadd:
3102     {
3103       RegisterOrConstant inc;
3104       Register tmp = as_reg(tmp_op);
3105       Register dst = as_reg(dest);
3106       if (data->is_constant()) {
3107         inc = RegisterOrConstant(as_long(data));
3108         assert_different_registers(dst, addr.base(), tmp,
3109                                    rscratch1, rscratch2);
3110       } else {
3111         inc = RegisterOrConstant(as_reg(data));
3112         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3113                                    rscratch1, rscratch2);
3114       }
3115       __ lea(tmp, addr);
3116       (_masm->*add)(dst, inc, tmp);
3117       break;
3118     }
3119   case lir_xchg:
3120     {
3121       Register tmp = tmp_op->as_register();
3122       Register obj = as_reg(data);
3123       Register dst = as_reg(dest);
3124       if (is_oop && UseCompressedOops) {
3125         __ encode_heap_oop(rscratch2, obj);
3126         obj = rscratch2;
3127       }
3128       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3129       __ lea(tmp, addr);
3130       (_masm->*xchg)(dst, obj, tmp);
3131       if (is_oop && UseCompressedOops) {
3132         __ decode_heap_oop(dst);
3133       }
3134     }
3135     break;
3136   default:
3137     ShouldNotReachHere();
3138   }
3139   __ membar(__ AnyAny);
3140 }
3141 
3142 #undef __