1 /*
   2  * Copyright (c) 2000, 2019, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include "precompiled.hpp"
  27 #include "asm/macroAssembler.inline.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Compilation.hpp"
  31 #include "c1/c1_LIRAssembler.hpp"
  32 #include "c1/c1_MacroAssembler.hpp"
  33 #include "c1/c1_Runtime1.hpp"
  34 #include "c1/c1_ValueStack.hpp"
  35 #include "ci/ciArrayKlass.hpp"
  36 #include "ci/ciInstance.hpp"
  37 #include "code/compiledIC.hpp"
  38 #include "gc/shared/barrierSet.hpp"
  39 #include "gc/shared/cardTableBarrierSet.hpp"
  40 #include "gc/shared/collectedHeap.hpp"
  41 #include "nativeInst_aarch64.hpp"
  42 #include "oops/objArrayKlass.hpp"
  43 #include "runtime/frame.inline.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "vmreg_aarch64.inline.hpp"
  46 
  47 
  48 
  49 #ifndef PRODUCT
  50 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  51 #else
  52 #define COMMENT(x)
  53 #endif
  54 
  55 NEEDS_CLEANUP // remove this definitions ?
  56 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  57 const Register SYNC_header = r0;   // synchronization header
  58 const Register SHIFT_count = r0;   // where count for shift operations must be
  59 
  60 #define __ _masm->
  61 
  62 
  63 static void select_different_registers(Register preserve,
  64                                        Register extra,
  65                                        Register &tmp1,
  66                                        Register &tmp2) {
  67   if (tmp1 == preserve) {
  68     assert_different_registers(tmp1, tmp2, extra);
  69     tmp1 = extra;
  70   } else if (tmp2 == preserve) {
  71     assert_different_registers(tmp1, tmp2, extra);
  72     tmp2 = extra;
  73   }
  74   assert_different_registers(preserve, tmp1, tmp2);
  75 }
  76 
  77 
  78 
  79 static void select_different_registers(Register preserve,
  80                                        Register extra,
  81                                        Register &tmp1,
  82                                        Register &tmp2,
  83                                        Register &tmp3) {
  84   if (tmp1 == preserve) {
  85     assert_different_registers(tmp1, tmp2, tmp3, extra);
  86     tmp1 = extra;
  87   } else if (tmp2 == preserve) {
  88     assert_different_registers(tmp1, tmp2, tmp3, extra);
  89     tmp2 = extra;
  90   } else if (tmp3 == preserve) {
  91     assert_different_registers(tmp1, tmp2, tmp3, extra);
  92     tmp3 = extra;
  93   }
  94   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  95 }
  96 
  97 
  98 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
  99 
 100 
 101 LIR_Opr LIR_Assembler::receiverOpr() {
 102   return FrameMap::receiver_opr;
 103 }
 104 
 105 LIR_Opr LIR_Assembler::osrBufferPointer() {
 106   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 107 }
 108 
 109 //--------------fpu register translations-----------------------
 110 
 111 
 112 address LIR_Assembler::float_constant(float f) {
 113   address const_addr = __ float_constant(f);
 114   if (const_addr == NULL) {
 115     bailout("const section overflow");
 116     return __ code()->consts()->start();
 117   } else {
 118     return const_addr;
 119   }
 120 }
 121 
 122 
 123 address LIR_Assembler::double_constant(double d) {
 124   address const_addr = __ double_constant(d);
 125   if (const_addr == NULL) {
 126     bailout("const section overflow");
 127     return __ code()->consts()->start();
 128   } else {
 129     return const_addr;
 130   }
 131 }
 132 
 133 address LIR_Assembler::int_constant(jlong n) {
 134   address const_addr = __ long_constant(n);
 135   if (const_addr == NULL) {
 136     bailout("const section overflow");
 137     return __ code()->consts()->start();
 138   } else {
 139     return const_addr;
 140   }
 141 }
 142 
 143 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
 144 
 145 void LIR_Assembler::reset_FPU() { Unimplemented(); }
 146 
 147 void LIR_Assembler::fpop() { Unimplemented(); }
 148 
 149 void LIR_Assembler::fxch(int i) { Unimplemented(); }
 150 
 151 void LIR_Assembler::fld(int i) { Unimplemented(); }
 152 
 153 void LIR_Assembler::ffree(int i) { Unimplemented(); }
 154 
 155 void LIR_Assembler::breakpoint() { Unimplemented(); }
 156 
 157 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 158 
 159 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 160 
 161 bool LIR_Assembler::is_literal_address(LIR_Address* addr) { Unimplemented(); return false; }
 162 //-------------------------------------------
 163 
 164 static Register as_reg(LIR_Opr op) {
 165   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 166 }
 167 
 168 static jlong as_long(LIR_Opr data) {
 169   jlong result;
 170   switch (data->type()) {
 171   case T_INT:
 172     result = (data->as_jint());
 173     break;
 174   case T_LONG:
 175     result = (data->as_jlong());
 176     break;
 177   default:
 178     ShouldNotReachHere();
 179     result = 0;  // unreachable
 180   }
 181   return result;
 182 }
 183 
 184 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) {
 185   Register base = addr->base()->as_pointer_register();
 186   LIR_Opr opr = addr->index();
 187   if (opr->is_cpu_register()) {
 188     Register index;
 189     if (opr->is_single_cpu())
 190       index = opr->as_register();
 191     else
 192       index = opr->as_register_lo();
 193     assert(addr->disp() == 0, "must be");
 194     switch(opr->type()) {
 195       case T_INT:
 196         return Address(base, index, Address::sxtw(addr->scale()));
 197       case T_LONG:
 198         return Address(base, index, Address::lsl(addr->scale()));
 199       default:
 200         ShouldNotReachHere();
 201       }
 202   } else  {
 203     intptr_t addr_offset = intptr_t(addr->disp());
 204     if (Address::offset_ok_for_immed(addr_offset, addr->scale()))
 205       return Address(base, addr_offset, Address::lsl(addr->scale()));
 206     else {
 207       __ mov(tmp, addr_offset);
 208       return Address(base, tmp, Address::lsl(addr->scale()));
 209     }
 210   }
 211   return Address();
 212 }
 213 
 214 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 215   ShouldNotReachHere();
 216   return Address();
 217 }
 218 
 219 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 220   return as_Address(addr, rscratch1);
 221 }
 222 
 223 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 224   return as_Address(addr, rscratch1);  // Ouch
 225   // FIXME: This needs to be much more clever.  See x86.
 226 }
 227 
 228 
 229 void LIR_Assembler::osr_entry() {
 230   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 231   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 232   ValueStack* entry_state = osr_entry->state();
 233   int number_of_locks = entry_state->locks_size();
 234 
 235   // we jump here if osr happens with the interpreter
 236   // state set up to continue at the beginning of the
 237   // loop that triggered osr - in particular, we have
 238   // the following registers setup:
 239   //
 240   // r2: osr buffer
 241   //
 242 
 243   // build frame
 244   ciMethod* m = compilation()->method();
 245   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 246 
 247   // OSR buffer is
 248   //
 249   // locals[nlocals-1..0]
 250   // monitors[0..number_of_locks]
 251   //
 252   // locals is a direct copy of the interpreter frame so in the osr buffer
 253   // so first slot in the local array is the last local from the interpreter
 254   // and last slot is local[0] (receiver) from the interpreter
 255   //
 256   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 257   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 258   // in the interpreter frame (the method lock if a sync method)
 259 
 260   // Initialize monitors in the compiled activation.
 261   //   r2: pointer to osr buffer
 262   //
 263   // All other registers are dead at this point and the locals will be
 264   // copied into place by code emitted in the IR.
 265 
 266   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 267   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 268     int monitor_offset = BytesPerWord * method()->max_locals() +
 269       (2 * BytesPerWord) * (number_of_locks - 1);
 270     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 271     // the OSR buffer using 2 word entries: first the lock and then
 272     // the oop.
 273     for (int i = 0; i < number_of_locks; i++) {
 274       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 275 #ifdef ASSERT
 276       // verify the interpreter's monitor has a non-null object
 277       {
 278         Label L;
 279         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 280         __ cbnz(rscratch1, L);
 281         __ stop("locked object is NULL");
 282         __ bind(L);
 283       }
 284 #endif
 285       __ ldr(r19, Address(OSR_buf, slot_offset + 0));
 286       __ str(r19, frame_map()->address_for_monitor_lock(i));
 287       __ ldr(r19, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 288       __ str(r19, frame_map()->address_for_monitor_object(i));
 289     }
 290   }
 291 }
 292 
 293 
 294 // inline cache check; done before the frame is built.
 295 int LIR_Assembler::check_icache() {
 296   Register receiver = FrameMap::receiver_opr->as_register();
 297   Register ic_klass = IC_Klass;
 298   int start_offset = __ offset();
 299   __ inline_cache_check(receiver, ic_klass);
 300 
 301   // if icache check fails, then jump to runtime routine
 302   // Note: RECEIVER must still contain the receiver!
 303   Label dont;
 304   __ br(Assembler::EQ, dont);
 305   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 306 
 307   // We align the verified entry point unless the method body
 308   // (including its inline cache check) will fit in a single 64-byte
 309   // icache line.
 310   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 311     // force alignment after the cache check.
 312     __ align(CodeEntryAlignment);
 313   }
 314 
 315   __ bind(dont);
 316   return start_offset;
 317 }
 318 
 319 
 320 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 321   if (o == NULL) {
 322     __ mov(reg, zr);
 323   } else {
 324     __ movoop(reg, o, /*immediate*/true);
 325   }
 326 }
 327 
 328 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 329   address target = NULL;
 330   relocInfo::relocType reloc_type = relocInfo::none;
 331 
 332   switch (patching_id(info)) {
 333   case PatchingStub::access_field_id:
 334     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 335     reloc_type = relocInfo::section_word_type;
 336     break;
 337   case PatchingStub::load_klass_id:
 338     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 339     reloc_type = relocInfo::metadata_type;
 340     break;
 341   case PatchingStub::load_mirror_id:
 342     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 343     reloc_type = relocInfo::oop_type;
 344     break;
 345   case PatchingStub::load_appendix_id:
 346     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 347     reloc_type = relocInfo::oop_type;
 348     break;
 349   default: ShouldNotReachHere();
 350   }
 351 
 352   __ far_call(RuntimeAddress(target));
 353   add_call_info_here(info);
 354 }
 355 
 356 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 357   deoptimize_trap(info);
 358 }
 359 
 360 
 361 // This specifies the rsp decrement needed to build the frame
 362 int LIR_Assembler::initial_frame_size_in_bytes() const {
 363   // if rounding, must let FrameMap know!
 364 
 365   // The frame_map records size in slots (32bit word)
 366 
 367   // subtract two words to account for return address and link
 368   return (frame_map()->framesize() - (2*VMRegImpl::slots_per_word))  * VMRegImpl::stack_slot_size;
 369 }
 370 
 371 
 372 int LIR_Assembler::emit_exception_handler() {
 373   // if the last instruction is a call (typically to do a throw which
 374   // is coming at the end after block reordering) the return address
 375   // must still point into the code area in order to avoid assertion
 376   // failures when searching for the corresponding bci => add a nop
 377   // (was bug 5/14/1999 - gri)
 378   __ nop();
 379 
 380   // generate code for exception handler
 381   address handler_base = __ start_a_stub(exception_handler_size());
 382   if (handler_base == NULL) {
 383     // not enough space left for the handler
 384     bailout("exception handler overflow");
 385     return -1;
 386   }
 387 
 388   int offset = code_offset();
 389 
 390   // the exception oop and pc are in r0, and r3
 391   // no other registers need to be preserved, so invalidate them
 392   __ invalidate_registers(false, true, true, false, true, true);
 393 
 394   // check that there is really an exception
 395   __ verify_not_null_oop(r0);
 396 
 397   // search an exception handler (r0: exception oop, r3: throwing pc)
 398   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));  __ should_not_reach_here();
 399   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 400   __ end_a_stub();
 401 
 402   return offset;
 403 }
 404 
 405 
 406 // Emit the code to remove the frame from the stack in the exception
 407 // unwind path.
 408 int LIR_Assembler::emit_unwind_handler() {
 409 #ifndef PRODUCT
 410   if (CommentedAssembly) {
 411     _masm->block_comment("Unwind handler");
 412   }
 413 #endif
 414 
 415   int offset = code_offset();
 416 
 417   // Fetch the exception from TLS and clear out exception related thread state
 418   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 419   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
 420   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
 421 
 422   __ bind(_unwind_handler_entry);
 423   __ verify_not_null_oop(r0);
 424   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 425     __ mov(r19, r0);  // Preserve the exception
 426   }
 427 
 428   // Preform needed unlocking
 429   MonitorExitStub* stub = NULL;
 430   if (method()->is_synchronized()) {
 431     monitor_address(0, FrameMap::r0_opr);
 432     stub = new MonitorExitStub(FrameMap::r0_opr, true, 0);
 433     __ unlock_object(r5, r4, r0, *stub->entry());
 434     __ bind(*stub->continuation());
 435   }
 436 
 437   if (compilation()->env()->dtrace_method_probes()) {
 438     __ call_Unimplemented();
 439 #if 0
 440     __ movptr(Address(rsp, 0), rax);
 441     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
 442     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 443 #endif
 444   }
 445 
 446   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 447     __ mov(r0, r19);  // Restore the exception
 448   }
 449 
 450   // remove the activation and dispatch to the unwind handler
 451   __ block_comment("remove_frame and dispatch to the unwind handler");
 452   __ remove_frame(initial_frame_size_in_bytes());
 453   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 454 
 455   // Emit the slow path assembly
 456   if (stub != NULL) {
 457     stub->emit_code(this);
 458   }
 459 
 460   return offset;
 461 }
 462 
 463 
 464 int LIR_Assembler::emit_deopt_handler() {
 465   // if the last instruction is a call (typically to do a throw which
 466   // is coming at the end after block reordering) the return address
 467   // must still point into the code area in order to avoid assertion
 468   // failures when searching for the corresponding bci => add a nop
 469   // (was bug 5/14/1999 - gri)
 470   __ nop();
 471 
 472   // generate code for exception handler
 473   address handler_base = __ start_a_stub(deopt_handler_size());
 474   if (handler_base == NULL) {
 475     // not enough space left for the handler
 476     bailout("deopt handler overflow");
 477     return -1;
 478   }
 479 
 480   int offset = code_offset();
 481 
 482   __ adr(lr, pc());
 483   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 484   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 485   __ end_a_stub();
 486 
 487   return offset;
 488 }
 489 
 490 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 491   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 492   int pc_offset = code_offset();
 493   flush_debug_info(pc_offset);
 494   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 495   if (info->exception_handlers() != NULL) {
 496     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 497   }
 498 }
 499 
 500 void LIR_Assembler::return_op(LIR_Opr result) {
 501   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 502 
 503   // Pop the stack before the safepoint code
 504   __ remove_frame(initial_frame_size_in_bytes());
 505 
 506   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 507     __ reserved_stack_check();
 508   }
 509 
 510   address polling_page(os::get_polling_page());
 511   __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
 512   __ ret(lr);
 513 }
 514 
 515 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 516   address polling_page(os::get_polling_page());
 517   guarantee(info != NULL, "Shouldn't be NULL");
 518   assert(os::is_poll_address(polling_page), "should be");
 519   __ get_polling_page(rscratch1, polling_page, relocInfo::poll_type);
 520   add_debug_info_for_branch(info);  // This isn't just debug info:
 521                                     // it's the oop map
 522   __ read_polling_page(rscratch1, relocInfo::poll_type);
 523   return __ offset();
 524 }
 525 
 526 
 527 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 528   if (from_reg == r31_sp)
 529     from_reg = sp;
 530   if (to_reg == r31_sp)
 531     to_reg = sp;
 532   __ mov(to_reg, from_reg);
 533 }
 534 
 535 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); }
 536 
 537 
 538 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 539   assert(src->is_constant(), "should not call otherwise");
 540   assert(dest->is_register(), "should not call otherwise");
 541   LIR_Const* c = src->as_constant_ptr();
 542 
 543   switch (c->type()) {
 544     case T_INT: {
 545       assert(patch_code == lir_patch_none, "no patching handled here");
 546       __ movw(dest->as_register(), c->as_jint());
 547       break;
 548     }
 549 
 550     case T_ADDRESS: {
 551       assert(patch_code == lir_patch_none, "no patching handled here");
 552       __ mov(dest->as_register(), c->as_jint());
 553       break;
 554     }
 555 
 556     case T_LONG: {
 557       assert(patch_code == lir_patch_none, "no patching handled here");
 558       __ mov(dest->as_register_lo(), (intptr_t)c->as_jlong());
 559       break;
 560     }
 561 
 562     case T_OBJECT: {
 563         if (patch_code == lir_patch_none) {
 564           jobject2reg(c->as_jobject(), dest->as_register());
 565         } else {
 566           jobject2reg_with_patching(dest->as_register(), info);
 567         }
 568       break;
 569     }
 570 
 571     case T_METADATA: {
 572       if (patch_code != lir_patch_none) {
 573         klass2reg_with_patching(dest->as_register(), info);
 574       } else {
 575         __ mov_metadata(dest->as_register(), c->as_metadata());
 576       }
 577       break;
 578     }
 579 
 580     case T_FLOAT: {
 581       if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 582         __ fmovs(dest->as_float_reg(), (c->as_jfloat()));
 583       } else {
 584         __ adr(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 585         __ ldrs(dest->as_float_reg(), Address(rscratch1));
 586       }
 587       break;
 588     }
 589 
 590     case T_DOUBLE: {
 591       if (__ operand_valid_for_float_immediate(c->as_jdouble())) {
 592         __ fmovd(dest->as_double_reg(), (c->as_jdouble()));
 593       } else {
 594         __ adr(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 595         __ ldrd(dest->as_double_reg(), Address(rscratch1));
 596       }
 597       break;
 598     }
 599 
 600     default:
 601       ShouldNotReachHere();
 602   }
 603 }
 604 
 605 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 606   LIR_Const* c = src->as_constant_ptr();
 607   switch (c->type()) {
 608   case T_OBJECT:
 609     {
 610       if (! c->as_jobject())
 611         __ str(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 612       else {
 613         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 614         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 615       }
 616     }
 617     break;
 618   case T_ADDRESS:
 619     {
 620       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 621       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 622     }
 623   case T_INT:
 624   case T_FLOAT:
 625     {
 626       Register reg = zr;
 627       if (c->as_jint_bits() == 0)
 628         __ strw(zr, frame_map()->address_for_slot(dest->single_stack_ix()));
 629       else {
 630         __ movw(rscratch1, c->as_jint_bits());
 631         __ strw(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 632       }
 633     }
 634     break;
 635   case T_LONG:
 636   case T_DOUBLE:
 637     {
 638       Register reg = zr;
 639       if (c->as_jlong_bits() == 0)
 640         __ str(zr, frame_map()->address_for_slot(dest->double_stack_ix(),
 641                                                  lo_word_offset_in_bytes));
 642       else {
 643         __ mov(rscratch1, (intptr_t)c->as_jlong_bits());
 644         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 645                                                         lo_word_offset_in_bytes));
 646       }
 647     }
 648     break;
 649   default:
 650     ShouldNotReachHere();
 651   }
 652 }
 653 
 654 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 655   assert(src->is_constant(), "should not call otherwise");
 656   LIR_Const* c = src->as_constant_ptr();
 657   LIR_Address* to_addr = dest->as_address_ptr();
 658 
 659   void (Assembler::* insn)(Register Rt, const Address &adr);
 660 
 661   switch (type) {
 662   case T_ADDRESS:
 663     assert(c->as_jint() == 0, "should be");
 664     insn = &Assembler::str;
 665     break;
 666   case T_LONG:
 667     assert(c->as_jlong() == 0, "should be");
 668     insn = &Assembler::str;
 669     break;
 670   case T_INT:
 671     assert(c->as_jint() == 0, "should be");
 672     insn = &Assembler::strw;
 673     break;
 674   case T_OBJECT:
 675   case T_ARRAY:
 676     assert(c->as_jobject() == 0, "should be");
 677     if (UseCompressedOops && !wide) {
 678       insn = &Assembler::strw;
 679     } else {
 680       insn = &Assembler::str;
 681     }
 682     break;
 683   case T_CHAR:
 684   case T_SHORT:
 685     assert(c->as_jint() == 0, "should be");
 686     insn = &Assembler::strh;
 687     break;
 688   case T_BOOLEAN:
 689   case T_BYTE:
 690     assert(c->as_jint() == 0, "should be");
 691     insn = &Assembler::strb;
 692     break;
 693   default:
 694     ShouldNotReachHere();
 695     insn = &Assembler::str;  // unreachable
 696   }
 697 
 698   if (info) add_debug_info_for_null_check_here(info);
 699   (_masm->*insn)(zr, as_Address(to_addr, rscratch1));
 700 }
 701 
 702 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 703   assert(src->is_register(), "should not call otherwise");
 704   assert(dest->is_register(), "should not call otherwise");
 705 
 706   // move between cpu-registers
 707   if (dest->is_single_cpu()) {
 708     if (src->type() == T_LONG) {
 709       // Can do LONG -> OBJECT
 710       move_regs(src->as_register_lo(), dest->as_register());
 711       return;
 712     }
 713     assert(src->is_single_cpu(), "must match");
 714     if (src->type() == T_OBJECT) {
 715       __ verify_oop(src->as_register());
 716     }
 717     move_regs(src->as_register(), dest->as_register());
 718 
 719   } else if (dest->is_double_cpu()) {
 720     if (src->type() == T_OBJECT || src->type() == T_ARRAY) {
 721       // Surprising to me but we can see move of a long to t_object
 722       __ verify_oop(src->as_register());
 723       move_regs(src->as_register(), dest->as_register_lo());
 724       return;
 725     }
 726     assert(src->is_double_cpu(), "must match");
 727     Register f_lo = src->as_register_lo();
 728     Register f_hi = src->as_register_hi();
 729     Register t_lo = dest->as_register_lo();
 730     Register t_hi = dest->as_register_hi();
 731     assert(f_hi == f_lo, "must be same");
 732     assert(t_hi == t_lo, "must be same");
 733     move_regs(f_lo, t_lo);
 734 
 735   } else if (dest->is_single_fpu()) {
 736     __ fmovs(dest->as_float_reg(), src->as_float_reg());
 737 
 738   } else if (dest->is_double_fpu()) {
 739     __ fmovd(dest->as_double_reg(), src->as_double_reg());
 740 
 741   } else {
 742     ShouldNotReachHere();
 743   }
 744 }
 745 
 746 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 747   if (src->is_single_cpu()) {
 748     if (type == T_ARRAY || type == T_OBJECT) {
 749       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 750       __ verify_oop(src->as_register());
 751     } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) {
 752       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 753     } else {
 754       __ strw(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 755     }
 756 
 757   } else if (src->is_double_cpu()) {
 758     Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 759     __ str(src->as_register_lo(), dest_addr_LO);
 760 
 761   } else if (src->is_single_fpu()) {
 762     Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 763     __ strs(src->as_float_reg(), dest_addr);
 764 
 765   } else if (src->is_double_fpu()) {
 766     Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 767     __ strd(src->as_double_reg(), dest_addr);
 768 
 769   } else {
 770     ShouldNotReachHere();
 771   }
 772 
 773 }
 774 
 775 
 776 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 777   LIR_Address* to_addr = dest->as_address_ptr();
 778   PatchingStub* patch = NULL;
 779   Register compressed_src = rscratch1;
 780 
 781   if (patch_code != lir_patch_none) {
 782     deoptimize_trap(info);
 783     return;
 784   }
 785 
 786   if (type == T_ARRAY || type == T_OBJECT) {
 787     __ verify_oop(src->as_register());
 788 
 789     if (UseCompressedOops && !wide) {
 790       __ encode_heap_oop(compressed_src, src->as_register());
 791     } else {
 792       compressed_src = src->as_register();
 793     }
 794   }
 795 
 796   int null_check_here = code_offset();
 797   switch (type) {
 798     case T_FLOAT: {
 799       __ strs(src->as_float_reg(), as_Address(to_addr));
 800       break;
 801     }
 802 
 803     case T_DOUBLE: {
 804       __ strd(src->as_double_reg(), as_Address(to_addr));
 805       break;
 806     }
 807 
 808     case T_ARRAY:   // fall through
 809     case T_OBJECT:  // fall through
 810       if (UseCompressedOops && !wide) {
 811         __ strw(compressed_src, as_Address(to_addr, rscratch2));
 812       } else {
 813          __ str(compressed_src, as_Address(to_addr));
 814       }
 815       break;
 816     case T_METADATA:
 817       // We get here to store a method pointer to the stack to pass to
 818       // a dtrace runtime call. This can't work on 64 bit with
 819       // compressed klass ptrs: T_METADATA can be a compressed klass
 820       // ptr or a 64 bit method pointer.
 821       ShouldNotReachHere();
 822       __ str(src->as_register(), as_Address(to_addr));
 823       break;
 824     case T_ADDRESS:
 825       __ str(src->as_register(), as_Address(to_addr));
 826       break;
 827     case T_INT:
 828       __ strw(src->as_register(), as_Address(to_addr));
 829       break;
 830 
 831     case T_LONG: {
 832       __ str(src->as_register_lo(), as_Address_lo(to_addr));
 833       break;
 834     }
 835 
 836     case T_BYTE:    // fall through
 837     case T_BOOLEAN: {
 838       __ strb(src->as_register(), as_Address(to_addr));
 839       break;
 840     }
 841 
 842     case T_CHAR:    // fall through
 843     case T_SHORT:
 844       __ strh(src->as_register(), as_Address(to_addr));
 845       break;
 846 
 847     default:
 848       ShouldNotReachHere();
 849   }
 850   if (info != NULL) {
 851     add_debug_info_for_null_check(null_check_here, info);
 852   }
 853 }
 854 
 855 
 856 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 857   assert(src->is_stack(), "should not call otherwise");
 858   assert(dest->is_register(), "should not call otherwise");
 859 
 860   if (dest->is_single_cpu()) {
 861     if (type == T_ARRAY || type == T_OBJECT) {
 862       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 863       __ verify_oop(dest->as_register());
 864     } else if (type == T_METADATA || type == T_ADDRESS) {
 865       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 866     } else {
 867       __ ldrw(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 868     }
 869 
 870   } else if (dest->is_double_cpu()) {
 871     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 872     __ ldr(dest->as_register_lo(), src_addr_LO);
 873 
 874   } else if (dest->is_single_fpu()) {
 875     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 876     __ ldrs(dest->as_float_reg(), src_addr);
 877 
 878   } else if (dest->is_double_fpu()) {
 879     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 880     __ ldrd(dest->as_double_reg(), src_addr);
 881 
 882   } else {
 883     ShouldNotReachHere();
 884   }
 885 }
 886 
 887 
 888 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 889   address target = NULL;
 890   relocInfo::relocType reloc_type = relocInfo::none;
 891 
 892   switch (patching_id(info)) {
 893   case PatchingStub::access_field_id:
 894     target = Runtime1::entry_for(Runtime1::access_field_patching_id);
 895     reloc_type = relocInfo::section_word_type;
 896     break;
 897   case PatchingStub::load_klass_id:
 898     target = Runtime1::entry_for(Runtime1::load_klass_patching_id);
 899     reloc_type = relocInfo::metadata_type;
 900     break;
 901   case PatchingStub::load_mirror_id:
 902     target = Runtime1::entry_for(Runtime1::load_mirror_patching_id);
 903     reloc_type = relocInfo::oop_type;
 904     break;
 905   case PatchingStub::load_appendix_id:
 906     target = Runtime1::entry_for(Runtime1::load_appendix_patching_id);
 907     reloc_type = relocInfo::oop_type;
 908     break;
 909   default: ShouldNotReachHere();
 910   }
 911 
 912   __ far_call(RuntimeAddress(target));
 913   add_call_info_here(info);
 914 }
 915 
 916 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 917 
 918   LIR_Opr temp;
 919   if (type == T_LONG || type == T_DOUBLE)
 920     temp = FrameMap::rscratch1_long_opr;
 921   else
 922     temp = FrameMap::rscratch1_opr;
 923 
 924   stack2reg(src, temp, src->type());
 925   reg2stack(temp, dest, dest->type(), false);
 926 }
 927 
 928 
 929 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
 930   LIR_Address* addr = src->as_address_ptr();
 931   LIR_Address* from_addr = src->as_address_ptr();
 932 
 933   if (addr->base()->type() == T_OBJECT) {
 934     __ verify_oop(addr->base()->as_pointer_register());
 935   }
 936 
 937   if (patch_code != lir_patch_none) {
 938     deoptimize_trap(info);
 939     return;
 940   }
 941 
 942   if (info != NULL) {
 943     add_debug_info_for_null_check_here(info);
 944   }
 945   int null_check_here = code_offset();
 946   switch (type) {
 947     case T_FLOAT: {
 948       __ ldrs(dest->as_float_reg(), as_Address(from_addr));
 949       break;
 950     }
 951 
 952     case T_DOUBLE: {
 953       __ ldrd(dest->as_double_reg(), as_Address(from_addr));
 954       break;
 955     }
 956 
 957     case T_ARRAY:   // fall through
 958     case T_OBJECT:  // fall through
 959       if (UseCompressedOops && !wide) {
 960         __ ldrw(dest->as_register(), as_Address(from_addr));
 961       } else {
 962          __ ldr(dest->as_register(), as_Address(from_addr));
 963       }
 964       break;
 965     case T_METADATA:
 966       // We get here to store a method pointer to the stack to pass to
 967       // a dtrace runtime call. This can't work on 64 bit with
 968       // compressed klass ptrs: T_METADATA can be a compressed klass
 969       // ptr or a 64 bit method pointer.
 970       ShouldNotReachHere();
 971       __ ldr(dest->as_register(), as_Address(from_addr));
 972       break;
 973     case T_ADDRESS:
 974       // FIXME: OMG this is a horrible kludge.  Any offset from an
 975       // address that matches klass_offset_in_bytes() will be loaded
 976       // as a word, not a long.
 977       if (UseCompressedClassPointers && addr->disp() == oopDesc::klass_offset_in_bytes()) {
 978         __ ldrw(dest->as_register(), as_Address(from_addr));
 979       } else {
 980         __ ldr(dest->as_register(), as_Address(from_addr));
 981       }
 982       break;
 983     case T_INT:
 984       __ ldrw(dest->as_register(), as_Address(from_addr));
 985       break;
 986 
 987     case T_LONG: {
 988       __ ldr(dest->as_register_lo(), as_Address_lo(from_addr));
 989       break;
 990     }
 991 
 992     case T_BYTE:
 993       __ ldrsb(dest->as_register(), as_Address(from_addr));
 994       break;
 995     case T_BOOLEAN: {
 996       __ ldrb(dest->as_register(), as_Address(from_addr));
 997       break;
 998     }
 999 
1000     case T_CHAR:
1001       __ ldrh(dest->as_register(), as_Address(from_addr));
1002       break;
1003     case T_SHORT:
1004       __ ldrsh(dest->as_register(), as_Address(from_addr));
1005       break;
1006 
1007     default:
1008       ShouldNotReachHere();
1009   }
1010 
1011   if (type == T_ARRAY || type == T_OBJECT) {
1012     if (UseCompressedOops && !wide) {
1013       __ decode_heap_oop(dest->as_register());
1014     }
1015     __ verify_oop(dest->as_register());
1016   } else if (type == T_ADDRESS && addr->disp() == oopDesc::klass_offset_in_bytes()) {
1017     if (UseCompressedClassPointers) {
1018       __ decode_klass_not_null(dest->as_register());
1019     }
1020   }
1021 }
1022 
1023 
1024 int LIR_Assembler::array_element_size(BasicType type) const {
1025   int elem_size = type2aelembytes(type);
1026   return exact_log2(elem_size);
1027 }
1028 
1029 void LIR_Assembler::arithmetic_idiv(LIR_Op3* op, bool is_irem) {
1030   Register Rdividend = op->in_opr1()->as_register();
1031   Register Rdivisor  = op->in_opr2()->as_register();
1032   Register Rscratch  = op->in_opr3()->as_register();
1033   Register Rresult   = op->result_opr()->as_register();
1034   int divisor = -1;
1035 
1036   /*
1037   TODO: For some reason, using the Rscratch that gets passed in is
1038   not possible because the register allocator does not see the tmp reg
1039   as used, and assignes it the same register as Rdividend. We use rscratch1
1040    instead.
1041 
1042   assert(Rdividend != Rscratch, "");
1043   assert(Rdivisor  != Rscratch, "");
1044   */
1045 
1046   if (Rdivisor == noreg && is_power_of_2(divisor)) {
1047     // convert division by a power of two into some shifts and logical operations
1048   }
1049 
1050   __ corrected_idivl(Rresult, Rdividend, Rdivisor, is_irem, rscratch1);
1051 }
1052 
1053 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1054   switch (op->code()) {
1055   case lir_idiv:
1056     arithmetic_idiv(op, false);
1057     break;
1058   case lir_irem:
1059     arithmetic_idiv(op, true);
1060     break;
1061   case lir_fmad:
1062     __ fmaddd(op->result_opr()->as_double_reg(),
1063               op->in_opr1()->as_double_reg(),
1064               op->in_opr2()->as_double_reg(),
1065               op->in_opr3()->as_double_reg());
1066     break;
1067   case lir_fmaf:
1068     __ fmadds(op->result_opr()->as_float_reg(),
1069               op->in_opr1()->as_float_reg(),
1070               op->in_opr2()->as_float_reg(),
1071               op->in_opr3()->as_float_reg());
1072     break;
1073   default:      ShouldNotReachHere(); break;
1074   }
1075 }
1076 
1077 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1078 #ifdef ASSERT
1079   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1080   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1081   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1082 #endif
1083 
1084   if (op->cond() == lir_cond_always) {
1085     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1086     __ b(*(op->label()));
1087   } else {
1088     Assembler::Condition acond;
1089     if (op->code() == lir_cond_float_branch) {
1090       bool is_unordered = (op->ublock() == op->block());
1091       // Assembler::EQ does not permit unordered branches, so we add
1092       // another branch here.  Likewise, Assembler::NE does not permit
1093       // ordered branches.
1094       if ((is_unordered && op->cond() == lir_cond_equal)
1095           || (!is_unordered && op->cond() == lir_cond_notEqual))
1096         __ br(Assembler::VS, *(op->ublock()->label()));
1097       switch(op->cond()) {
1098       case lir_cond_equal:        acond = Assembler::EQ; break;
1099       case lir_cond_notEqual:     acond = Assembler::NE; break;
1100       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1101       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1102       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1103       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1104       default:                    ShouldNotReachHere();
1105         acond = Assembler::EQ;  // unreachable
1106       }
1107     } else {
1108       switch (op->cond()) {
1109         case lir_cond_equal:        acond = Assembler::EQ; break;
1110         case lir_cond_notEqual:     acond = Assembler::NE; break;
1111         case lir_cond_less:         acond = Assembler::LT; break;
1112         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1113         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1114         case lir_cond_greater:      acond = Assembler::GT; break;
1115         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1116         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1117         default:                    ShouldNotReachHere();
1118           acond = Assembler::EQ;  // unreachable
1119       }
1120     }
1121     __ br(acond,*(op->label()));
1122   }
1123 }
1124 
1125 
1126 
1127 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1128   LIR_Opr src  = op->in_opr();
1129   LIR_Opr dest = op->result_opr();
1130 
1131   switch (op->bytecode()) {
1132     case Bytecodes::_i2f:
1133       {
1134         __ scvtfws(dest->as_float_reg(), src->as_register());
1135         break;
1136       }
1137     case Bytecodes::_i2d:
1138       {
1139         __ scvtfwd(dest->as_double_reg(), src->as_register());
1140         break;
1141       }
1142     case Bytecodes::_l2d:
1143       {
1144         __ scvtfd(dest->as_double_reg(), src->as_register_lo());
1145         break;
1146       }
1147     case Bytecodes::_l2f:
1148       {
1149         __ scvtfs(dest->as_float_reg(), src->as_register_lo());
1150         break;
1151       }
1152     case Bytecodes::_f2d:
1153       {
1154         __ fcvts(dest->as_double_reg(), src->as_float_reg());
1155         break;
1156       }
1157     case Bytecodes::_d2f:
1158       {
1159         __ fcvtd(dest->as_float_reg(), src->as_double_reg());
1160         break;
1161       }
1162     case Bytecodes::_i2c:
1163       {
1164         __ ubfx(dest->as_register(), src->as_register(), 0, 16);
1165         break;
1166       }
1167     case Bytecodes::_i2l:
1168       {
1169         __ sxtw(dest->as_register_lo(), src->as_register());
1170         break;
1171       }
1172     case Bytecodes::_i2s:
1173       {
1174         __ sxth(dest->as_register(), src->as_register());
1175         break;
1176       }
1177     case Bytecodes::_i2b:
1178       {
1179         __ sxtb(dest->as_register(), src->as_register());
1180         break;
1181       }
1182     case Bytecodes::_l2i:
1183       {
1184         _masm->block_comment("FIXME: This could be a no-op");
1185         __ uxtw(dest->as_register(), src->as_register_lo());
1186         break;
1187       }
1188     case Bytecodes::_d2l:
1189       {
1190         __ fcvtzd(dest->as_register_lo(), src->as_double_reg());
1191         break;
1192       }
1193     case Bytecodes::_f2i:
1194       {
1195         __ fcvtzsw(dest->as_register(), src->as_float_reg());
1196         break;
1197       }
1198     case Bytecodes::_f2l:
1199       {
1200         __ fcvtzs(dest->as_register_lo(), src->as_float_reg());
1201         break;
1202       }
1203     case Bytecodes::_d2i:
1204       {
1205         __ fcvtzdw(dest->as_register(), src->as_double_reg());
1206         break;
1207       }
1208     default: ShouldNotReachHere();
1209   }
1210 }
1211 
1212 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1213   if (op->init_check()) {
1214     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1215                                InstanceKlass::init_state_offset()));
1216     __ cmpw(rscratch1, InstanceKlass::fully_initialized);
1217     add_debug_info_for_null_check_here(op->stub()->info());
1218     __ br(Assembler::NE, *op->stub()->entry());
1219   }
1220   __ allocate_object(op->obj()->as_register(),
1221                      op->tmp1()->as_register(),
1222                      op->tmp2()->as_register(),
1223                      op->header_size(),
1224                      op->object_size(),
1225                      op->klass()->as_register(),
1226                      *op->stub()->entry());
1227   __ bind(*op->stub()->continuation());
1228 }
1229 
1230 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1231   Register len =  op->len()->as_register();
1232   __ uxtw(len, len);
1233 
1234   if (UseSlowPath ||
1235       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1236       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1237     __ b(*op->stub()->entry());
1238   } else {
1239     Register tmp1 = op->tmp1()->as_register();
1240     Register tmp2 = op->tmp2()->as_register();
1241     Register tmp3 = op->tmp3()->as_register();
1242     if (len == tmp1) {
1243       tmp1 = tmp3;
1244     } else if (len == tmp2) {
1245       tmp2 = tmp3;
1246     } else if (len == tmp3) {
1247       // everything is ok
1248     } else {
1249       __ mov(tmp3, len);
1250     }
1251     __ allocate_array(op->obj()->as_register(),
1252                       len,
1253                       tmp1,
1254                       tmp2,
1255                       arrayOopDesc::header_size(op->type()),
1256                       array_element_size(op->type()),
1257                       op->klass()->as_register(),
1258                       *op->stub()->entry());
1259   }
1260   __ bind(*op->stub()->continuation());
1261 }
1262 
1263 void LIR_Assembler::type_profile_helper(Register mdo,
1264                                         ciMethodData *md, ciProfileData *data,
1265                                         Register recv, Label* update_done) {
1266   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1267     Label next_test;
1268     // See if the receiver is receiver[n].
1269     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1270     __ ldr(rscratch1, Address(rscratch2));
1271     __ cmp(recv, rscratch1);
1272     __ br(Assembler::NE, next_test);
1273     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1274     __ addptr(data_addr, DataLayout::counter_increment);
1275     __ b(*update_done);
1276     __ bind(next_test);
1277   }
1278 
1279   // Didn't find receiver; find next empty slot and fill it in
1280   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1281     Label next_test;
1282     __ lea(rscratch2,
1283            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1284     Address recv_addr(rscratch2);
1285     __ ldr(rscratch1, recv_addr);
1286     __ cbnz(rscratch1, next_test);
1287     __ str(recv, recv_addr);
1288     __ mov(rscratch1, DataLayout::counter_increment);
1289     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1290     __ str(rscratch1, Address(rscratch2));
1291     __ b(*update_done);
1292     __ bind(next_test);
1293   }
1294 }
1295 
1296 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1297   // we always need a stub for the failure case.
1298   CodeStub* stub = op->stub();
1299   Register obj = op->object()->as_register();
1300   Register k_RInfo = op->tmp1()->as_register();
1301   Register klass_RInfo = op->tmp2()->as_register();
1302   Register dst = op->result_opr()->as_register();
1303   ciKlass* k = op->klass();
1304   Register Rtmp1 = noreg;
1305 
1306   // check if it needs to be profiled
1307   ciMethodData* md;
1308   ciProfileData* data;
1309 
1310   const bool should_profile = op->should_profile();
1311 
1312   if (should_profile) {
1313     ciMethod* method = op->profiled_method();
1314     assert(method != NULL, "Should have method");
1315     int bci = op->profiled_bci();
1316     md = method->method_data_or_null();
1317     assert(md != NULL, "Sanity");
1318     data = md->bci_to_data(bci);
1319     assert(data != NULL,                "need data for type check");
1320     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1321   }
1322   Label profile_cast_success, profile_cast_failure;
1323   Label *success_target = should_profile ? &profile_cast_success : success;
1324   Label *failure_target = should_profile ? &profile_cast_failure : failure;
1325 
1326   if (obj == k_RInfo) {
1327     k_RInfo = dst;
1328   } else if (obj == klass_RInfo) {
1329     klass_RInfo = dst;
1330   }
1331   if (k->is_loaded() && !UseCompressedClassPointers) {
1332     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1333   } else {
1334     Rtmp1 = op->tmp3()->as_register();
1335     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1336   }
1337 
1338   assert_different_registers(obj, k_RInfo, klass_RInfo);
1339 
1340     if (should_profile) {
1341       Label not_null;
1342       __ cbnz(obj, not_null);
1343       // Object is null; update MDO and exit
1344       Register mdo  = klass_RInfo;
1345       __ mov_metadata(mdo, md->constant_encoding());
1346       Address data_addr
1347         = __ form_address(rscratch2, mdo,
1348                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1349                           0);
1350       __ ldrb(rscratch1, data_addr);
1351       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1352       __ strb(rscratch1, data_addr);
1353       __ b(*obj_is_null);
1354       __ bind(not_null);
1355     } else {
1356       __ cbz(obj, *obj_is_null);
1357     }
1358 
1359   if (!k->is_loaded()) {
1360     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1361   } else {
1362     __ mov_metadata(k_RInfo, k->constant_encoding());
1363   }
1364   __ verify_oop(obj);
1365 
1366   if (op->fast_check()) {
1367     // get object class
1368     // not a safepoint as obj null check happens earlier
1369     __ load_klass(rscratch1, obj);
1370     __ cmp( rscratch1, k_RInfo);
1371 
1372     __ br(Assembler::NE, *failure_target);
1373     // successful cast, fall through to profile or jump
1374   } else {
1375     // get object class
1376     // not a safepoint as obj null check happens earlier
1377     __ load_klass(klass_RInfo, obj);
1378     if (k->is_loaded()) {
1379       // See if we get an immediate positive hit
1380       __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1381       __ cmp(k_RInfo, rscratch1);
1382       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1383         __ br(Assembler::NE, *failure_target);
1384         // successful cast, fall through to profile or jump
1385       } else {
1386         // See if we get an immediate positive hit
1387         __ br(Assembler::EQ, *success_target);
1388         // check for self
1389         __ cmp(klass_RInfo, k_RInfo);
1390         __ br(Assembler::EQ, *success_target);
1391 
1392         __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1393         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1394         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1395         // result is a boolean
1396         __ cbzw(klass_RInfo, *failure_target);
1397         // successful cast, fall through to profile or jump
1398       }
1399     } else {
1400       // perform the fast part of the checking logic
1401       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1402       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1403       __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1404       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1405       __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1406       // result is a boolean
1407       __ cbz(k_RInfo, *failure_target);
1408       // successful cast, fall through to profile or jump
1409     }
1410   }
1411   if (should_profile) {
1412     Register mdo  = klass_RInfo, recv = k_RInfo;
1413     __ bind(profile_cast_success);
1414     __ mov_metadata(mdo, md->constant_encoding());
1415     __ load_klass(recv, obj);
1416     Label update_done;
1417     type_profile_helper(mdo, md, data, recv, success);
1418     __ b(*success);
1419 
1420     __ bind(profile_cast_failure);
1421     __ mov_metadata(mdo, md->constant_encoding());
1422     Address counter_addr
1423       = __ form_address(rscratch2, mdo,
1424                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1425                         0);
1426     __ ldr(rscratch1, counter_addr);
1427     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1428     __ str(rscratch1, counter_addr);
1429     __ b(*failure);
1430   }
1431   __ b(*success);
1432 }
1433 
1434 
1435 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1436   const bool should_profile = op->should_profile();
1437 
1438   LIR_Code code = op->code();
1439   if (code == lir_store_check) {
1440     Register value = op->object()->as_register();
1441     Register array = op->array()->as_register();
1442     Register k_RInfo = op->tmp1()->as_register();
1443     Register klass_RInfo = op->tmp2()->as_register();
1444     Register Rtmp1 = op->tmp3()->as_register();
1445 
1446     CodeStub* stub = op->stub();
1447 
1448     // check if it needs to be profiled
1449     ciMethodData* md;
1450     ciProfileData* data;
1451 
1452     if (should_profile) {
1453       ciMethod* method = op->profiled_method();
1454       assert(method != NULL, "Should have method");
1455       int bci = op->profiled_bci();
1456       md = method->method_data_or_null();
1457       assert(md != NULL, "Sanity");
1458       data = md->bci_to_data(bci);
1459       assert(data != NULL,                "need data for type check");
1460       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1461     }
1462     Label profile_cast_success, profile_cast_failure, done;
1463     Label *success_target = should_profile ? &profile_cast_success : &done;
1464     Label *failure_target = should_profile ? &profile_cast_failure : stub->entry();
1465 
1466     if (should_profile) {
1467       Label not_null;
1468       __ cbnz(value, not_null);
1469       // Object is null; update MDO and exit
1470       Register mdo  = klass_RInfo;
1471       __ mov_metadata(mdo, md->constant_encoding());
1472       Address data_addr
1473         = __ form_address(rscratch2, mdo,
1474                           md->byte_offset_of_slot(data, DataLayout::flags_offset()),
1475                           0);
1476       __ ldrb(rscratch1, data_addr);
1477       __ orr(rscratch1, rscratch1, BitData::null_seen_byte_constant());
1478       __ strb(rscratch1, data_addr);
1479       __ b(done);
1480       __ bind(not_null);
1481     } else {
1482       __ cbz(value, done);
1483     }
1484 
1485     add_debug_info_for_null_check_here(op->info_for_exception());
1486     __ load_klass(k_RInfo, array);
1487     __ load_klass(klass_RInfo, value);
1488 
1489     // get instance klass (it's already uncompressed)
1490     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1491     // perform the fast part of the checking logic
1492     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1493     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1494     __ stp(klass_RInfo, k_RInfo, Address(__ pre(sp, -2 * wordSize)));
1495     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1496     __ ldp(k_RInfo, klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1497     // result is a boolean
1498     __ cbzw(k_RInfo, *failure_target);
1499     // fall through to the success case
1500 
1501     if (should_profile) {
1502       Register mdo  = klass_RInfo, recv = k_RInfo;
1503       __ bind(profile_cast_success);
1504       __ mov_metadata(mdo, md->constant_encoding());
1505       __ load_klass(recv, value);
1506       Label update_done;
1507       type_profile_helper(mdo, md, data, recv, &done);
1508       __ b(done);
1509 
1510       __ bind(profile_cast_failure);
1511       __ mov_metadata(mdo, md->constant_encoding());
1512       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1513       __ lea(rscratch2, counter_addr);
1514       __ ldr(rscratch1, Address(rscratch2));
1515       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1516       __ str(rscratch1, Address(rscratch2));
1517       __ b(*stub->entry());
1518     }
1519 
1520     __ bind(done);
1521   } else if (code == lir_checkcast) {
1522     Register obj = op->object()->as_register();
1523     Register dst = op->result_opr()->as_register();
1524     Label success;
1525     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1526     __ bind(success);
1527     if (dst != obj) {
1528       __ mov(dst, obj);
1529     }
1530   } else if (code == lir_instanceof) {
1531     Register obj = op->object()->as_register();
1532     Register dst = op->result_opr()->as_register();
1533     Label success, failure, done;
1534     emit_typecheck_helper(op, &success, &failure, &failure);
1535     __ bind(failure);
1536     __ mov(dst, zr);
1537     __ b(done);
1538     __ bind(success);
1539     __ mov(dst, 1);
1540     __ bind(done);
1541   } else {
1542     ShouldNotReachHere();
1543   }
1544 }
1545 
1546 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) {
1547   __ cmpxchg(addr, cmpval, newval, Assembler::word, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1548   __ cset(rscratch1, Assembler::NE);
1549   __ membar(__ AnyAny);
1550 }
1551 
1552 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) {
1553   __ cmpxchg(addr, cmpval, newval, Assembler::xword, /* acquire*/ true, /* release*/ true, /* weak*/ false, rscratch1);
1554   __ cset(rscratch1, Assembler::NE);
1555   __ membar(__ AnyAny);
1556 }
1557 
1558 
1559 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1560   assert(VM_Version::supports_cx8(), "wrong machine");
1561   Register addr;
1562   if (op->addr()->is_register()) {
1563     addr = as_reg(op->addr());
1564   } else {
1565     assert(op->addr()->is_address(), "what else?");
1566     LIR_Address* addr_ptr = op->addr()->as_address_ptr();
1567     assert(addr_ptr->disp() == 0, "need 0 disp");
1568     assert(addr_ptr->index() == LIR_OprDesc::illegalOpr(), "need 0 index");
1569     addr = as_reg(addr_ptr->base());
1570   }
1571   Register newval = as_reg(op->new_value());
1572   Register cmpval = as_reg(op->cmp_value());
1573   Label succeed, fail, around;
1574 
1575   if (op->code() == lir_cas_obj) {
1576     if (UseCompressedOops) {
1577       Register t1 = op->tmp1()->as_register();
1578       assert(op->tmp1()->is_valid(), "must be");
1579       __ encode_heap_oop(t1, cmpval);
1580       cmpval = t1;
1581       __ encode_heap_oop(rscratch2, newval);
1582       newval = rscratch2;
1583       casw(addr, newval, cmpval);
1584     } else {
1585       casl(addr, newval, cmpval);
1586     }
1587   } else if (op->code() == lir_cas_int) {
1588     casw(addr, newval, cmpval);
1589   } else {
1590     casl(addr, newval, cmpval);
1591   }
1592 }
1593 
1594 
1595 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1596 
1597   Assembler::Condition acond, ncond;
1598   switch (condition) {
1599   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1600   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1601   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1602   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1603   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1604   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1605   case lir_cond_belowEqual:
1606   case lir_cond_aboveEqual:
1607   default:                    ShouldNotReachHere();
1608     acond = Assembler::EQ; ncond = Assembler::NE;  // unreachable
1609   }
1610 
1611   assert(result->is_single_cpu() || result->is_double_cpu(),
1612          "expect single register for result");
1613   if (opr1->is_constant() && opr2->is_constant()
1614       && opr1->type() == T_INT && opr2->type() == T_INT) {
1615     jint val1 = opr1->as_jint();
1616     jint val2 = opr2->as_jint();
1617     if (val1 == 0 && val2 == 1) {
1618       __ cset(result->as_register(), ncond);
1619       return;
1620     } else if (val1 == 1 && val2 == 0) {
1621       __ cset(result->as_register(), acond);
1622       return;
1623     }
1624   }
1625 
1626   if (opr1->is_constant() && opr2->is_constant()
1627       && opr1->type() == T_LONG && opr2->type() == T_LONG) {
1628     jlong val1 = opr1->as_jlong();
1629     jlong val2 = opr2->as_jlong();
1630     if (val1 == 0 && val2 == 1) {
1631       __ cset(result->as_register_lo(), ncond);
1632       return;
1633     } else if (val1 == 1 && val2 == 0) {
1634       __ cset(result->as_register_lo(), acond);
1635       return;
1636     }
1637   }
1638 
1639   if (opr1->is_stack()) {
1640     stack2reg(opr1, FrameMap::rscratch1_opr, result->type());
1641     opr1 = FrameMap::rscratch1_opr;
1642   } else if (opr1->is_constant()) {
1643     LIR_Opr tmp
1644       = opr1->type() == T_LONG ? FrameMap::rscratch1_long_opr : FrameMap::rscratch1_opr;
1645     const2reg(opr1, tmp, lir_patch_none, NULL);
1646     opr1 = tmp;
1647   }
1648 
1649   if (opr2->is_stack()) {
1650     stack2reg(opr2, FrameMap::rscratch2_opr, result->type());
1651     opr2 = FrameMap::rscratch2_opr;
1652   } else if (opr2->is_constant()) {
1653     LIR_Opr tmp
1654       = opr2->type() == T_LONG ? FrameMap::rscratch2_long_opr : FrameMap::rscratch2_opr;
1655     const2reg(opr2, tmp, lir_patch_none, NULL);
1656     opr2 = tmp;
1657   }
1658 
1659   if (result->type() == T_LONG)
1660     __ csel(result->as_register_lo(), opr1->as_register_lo(), opr2->as_register_lo(), acond);
1661   else
1662     __ csel(result->as_register(), opr1->as_register(), opr2->as_register(), acond);
1663 }
1664 
1665 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1666   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1667 
1668   if (left->is_single_cpu()) {
1669     Register lreg = left->as_register();
1670     Register dreg = as_reg(dest);
1671 
1672     if (right->is_single_cpu()) {
1673       // cpu register - cpu register
1674 
1675       assert(left->type() == T_INT && right->type() == T_INT && dest->type() == T_INT,
1676              "should be");
1677       Register rreg = right->as_register();
1678       switch (code) {
1679       case lir_add: __ addw (dest->as_register(), lreg, rreg); break;
1680       case lir_sub: __ subw (dest->as_register(), lreg, rreg); break;
1681       case lir_mul: __ mulw (dest->as_register(), lreg, rreg); break;
1682       default:      ShouldNotReachHere();
1683       }
1684 
1685     } else if (right->is_double_cpu()) {
1686       Register rreg = right->as_register_lo();
1687       // single_cpu + double_cpu: can happen with obj+long
1688       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1689       switch (code) {
1690       case lir_add: __ add(dreg, lreg, rreg); break;
1691       case lir_sub: __ sub(dreg, lreg, rreg); break;
1692       default: ShouldNotReachHere();
1693       }
1694     } else if (right->is_constant()) {
1695       // cpu register - constant
1696       jlong c;
1697 
1698       // FIXME.  This is fugly: we really need to factor all this logic.
1699       switch(right->type()) {
1700       case T_LONG:
1701         c = right->as_constant_ptr()->as_jlong();
1702         break;
1703       case T_INT:
1704       case T_ADDRESS:
1705         c = right->as_constant_ptr()->as_jint();
1706         break;
1707       default:
1708         ShouldNotReachHere();
1709         c = 0;  // unreachable
1710         break;
1711       }
1712 
1713       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1714       if (c == 0 && dreg == lreg) {
1715         COMMENT("effective nop elided");
1716         return;
1717       }
1718       switch(left->type()) {
1719       case T_INT:
1720         switch (code) {
1721         case lir_add: __ addw(dreg, lreg, c); break;
1722         case lir_sub: __ subw(dreg, lreg, c); break;
1723         default: ShouldNotReachHere();
1724         }
1725         break;
1726       case T_OBJECT:
1727       case T_ADDRESS:
1728         switch (code) {
1729         case lir_add: __ add(dreg, lreg, c); break;
1730         case lir_sub: __ sub(dreg, lreg, c); break;
1731         default: ShouldNotReachHere();
1732         }
1733         break;
1734         ShouldNotReachHere();
1735       }
1736     } else {
1737       ShouldNotReachHere();
1738     }
1739 
1740   } else if (left->is_double_cpu()) {
1741     Register lreg_lo = left->as_register_lo();
1742 
1743     if (right->is_double_cpu()) {
1744       // cpu register - cpu register
1745       Register rreg_lo = right->as_register_lo();
1746       switch (code) {
1747       case lir_add: __ add (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1748       case lir_sub: __ sub (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1749       case lir_mul: __ mul (dest->as_register_lo(), lreg_lo, rreg_lo); break;
1750       case lir_div: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, false, rscratch1); break;
1751       case lir_rem: __ corrected_idivq(dest->as_register_lo(), lreg_lo, rreg_lo, true, rscratch1); break;
1752       default:
1753         ShouldNotReachHere();
1754       }
1755 
1756     } else if (right->is_constant()) {
1757       jlong c = right->as_constant_ptr()->as_jlong_bits();
1758       Register dreg = as_reg(dest);
1759       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1760       if (c == 0 && dreg == lreg_lo) {
1761         COMMENT("effective nop elided");
1762         return;
1763       }
1764       switch (code) {
1765         case lir_add: __ add(dreg, lreg_lo, c); break;
1766         case lir_sub: __ sub(dreg, lreg_lo, c); break;
1767         default:
1768           ShouldNotReachHere();
1769       }
1770     } else {
1771       ShouldNotReachHere();
1772     }
1773   } else if (left->is_single_fpu()) {
1774     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1775     switch (code) {
1776     case lir_add: __ fadds (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1777     case lir_sub: __ fsubs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1778     case lir_mul_strictfp: // fall through
1779     case lir_mul: __ fmuls (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1780     case lir_div_strictfp: // fall through
1781     case lir_div: __ fdivs (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1782     default:
1783       ShouldNotReachHere();
1784     }
1785   } else if (left->is_double_fpu()) {
1786     if (right->is_double_fpu()) {
1787       // fpu register - fpu register
1788       switch (code) {
1789       case lir_add: __ faddd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1790       case lir_sub: __ fsubd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1791       case lir_mul_strictfp: // fall through
1792       case lir_mul: __ fmuld (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1793       case lir_div_strictfp: // fall through
1794       case lir_div: __ fdivd (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1795       default:
1796         ShouldNotReachHere();
1797       }
1798     } else {
1799       if (right->is_constant()) {
1800         ShouldNotReachHere();
1801       }
1802       ShouldNotReachHere();
1803     }
1804   } else if (left->is_single_stack() || left->is_address()) {
1805     assert(left == dest, "left and dest must be equal");
1806     ShouldNotReachHere();
1807   } else {
1808     ShouldNotReachHere();
1809   }
1810 }
1811 
1812 void LIR_Assembler::arith_fpu_implementation(LIR_Code code, int left_index, int right_index, int dest_index, bool pop_fpu_stack) { Unimplemented(); }
1813 
1814 
1815 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1816   switch(code) {
1817   case lir_abs : __ fabsd(dest->as_double_reg(), value->as_double_reg()); break;
1818   case lir_sqrt: __ fsqrtd(dest->as_double_reg(), value->as_double_reg()); break;
1819   default      : ShouldNotReachHere();
1820   }
1821 }
1822 
1823 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1824 
1825   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1826   Register Rleft = left->is_single_cpu() ? left->as_register() :
1827                                            left->as_register_lo();
1828    if (dst->is_single_cpu()) {
1829      Register Rdst = dst->as_register();
1830      if (right->is_constant()) {
1831        switch (code) {
1832          case lir_logic_and: __ andw (Rdst, Rleft, right->as_jint()); break;
1833          case lir_logic_or:  __ orrw (Rdst, Rleft, right->as_jint()); break;
1834          case lir_logic_xor: __ eorw (Rdst, Rleft, right->as_jint()); break;
1835          default: ShouldNotReachHere(); break;
1836        }
1837      } else {
1838        Register Rright = right->is_single_cpu() ? right->as_register() :
1839                                                   right->as_register_lo();
1840        switch (code) {
1841          case lir_logic_and: __ andw (Rdst, Rleft, Rright); break;
1842          case lir_logic_or:  __ orrw (Rdst, Rleft, Rright); break;
1843          case lir_logic_xor: __ eorw (Rdst, Rleft, Rright); break;
1844          default: ShouldNotReachHere(); break;
1845        }
1846      }
1847    } else {
1848      Register Rdst = dst->as_register_lo();
1849      if (right->is_constant()) {
1850        switch (code) {
1851          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jlong()); break;
1852          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jlong()); break;
1853          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jlong()); break;
1854          default: ShouldNotReachHere(); break;
1855        }
1856      } else {
1857        Register Rright = right->is_single_cpu() ? right->as_register() :
1858                                                   right->as_register_lo();
1859        switch (code) {
1860          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1861          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
1862          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1863          default: ShouldNotReachHere(); break;
1864        }
1865      }
1866    }
1867 }
1868 
1869 
1870 
1871 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1872 
1873 
1874 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1875   if (opr1->is_constant() && opr2->is_single_cpu()) {
1876     // tableswitch
1877     Register reg = as_reg(opr2);
1878     struct tableswitch &table = switches[opr1->as_constant_ptr()->as_jint()];
1879     __ tableswitch(reg, table._first_key, table._last_key, table._branches, table._after);
1880   } else if (opr1->is_single_cpu() || opr1->is_double_cpu()) {
1881     Register reg1 = as_reg(opr1);
1882     if (opr2->is_single_cpu()) {
1883       // cpu register - cpu register
1884       Register reg2 = opr2->as_register();
1885       if (opr1->type() == T_OBJECT || opr1->type() == T_ARRAY) {
1886         __ cmpoop(reg1, reg2);
1887       } else {
1888         assert(opr2->type() != T_OBJECT && opr2->type() != T_ARRAY, "cmp int, oop?");
1889         __ cmpw(reg1, reg2);
1890       }
1891       return;
1892     }
1893     if (opr2->is_double_cpu()) {
1894       // cpu register - cpu register
1895       Register reg2 = opr2->as_register_lo();
1896       __ cmp(reg1, reg2);
1897       return;
1898     }
1899 
1900     if (opr2->is_constant()) {
1901       bool is_32bit = false; // width of register operand
1902       jlong imm;
1903 
1904       switch(opr2->type()) {
1905       case T_INT:
1906         imm = opr2->as_constant_ptr()->as_jint();
1907         is_32bit = true;
1908         break;
1909       case T_LONG:
1910         imm = opr2->as_constant_ptr()->as_jlong();
1911         break;
1912       case T_ADDRESS:
1913         imm = opr2->as_constant_ptr()->as_jint();
1914         break;
1915       case T_METADATA:
1916         imm = (intptr_t)(opr2->as_constant_ptr()->as_metadata());
1917         break;
1918       case T_OBJECT:
1919       case T_ARRAY:
1920         jobject2reg(opr2->as_constant_ptr()->as_jobject(), rscratch1);
1921         __ cmpoop(reg1, rscratch1);
1922         return;
1923       default:
1924         ShouldNotReachHere();
1925         imm = 0;  // unreachable
1926         break;
1927       }
1928 
1929       if (Assembler::operand_valid_for_add_sub_immediate(imm)) {
1930         if (is_32bit)
1931           __ cmpw(reg1, imm);
1932         else
1933           __ cmp(reg1, imm);
1934         return;
1935       } else {
1936         __ mov(rscratch1, imm);
1937         if (is_32bit)
1938           __ cmpw(reg1, rscratch1);
1939         else
1940           __ cmp(reg1, rscratch1);
1941         return;
1942       }
1943     } else
1944       ShouldNotReachHere();
1945   } else if (opr1->is_single_fpu()) {
1946     FloatRegister reg1 = opr1->as_float_reg();
1947     assert(opr2->is_single_fpu(), "expect single float register");
1948     FloatRegister reg2 = opr2->as_float_reg();
1949     __ fcmps(reg1, reg2);
1950   } else if (opr1->is_double_fpu()) {
1951     FloatRegister reg1 = opr1->as_double_reg();
1952     assert(opr2->is_double_fpu(), "expect double float register");
1953     FloatRegister reg2 = opr2->as_double_reg();
1954     __ fcmpd(reg1, reg2);
1955   } else {
1956     ShouldNotReachHere();
1957   }
1958 }
1959 
1960 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1961   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1962     bool is_unordered_less = (code == lir_ucmp_fd2i);
1963     if (left->is_single_fpu()) {
1964       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1965     } else if (left->is_double_fpu()) {
1966       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1967     } else {
1968       ShouldNotReachHere();
1969     }
1970   } else if (code == lir_cmp_l2i) {
1971     Label done;
1972     __ cmp(left->as_register_lo(), right->as_register_lo());
1973     __ mov(dst->as_register(), (u_int64_t)-1L);
1974     __ br(Assembler::LT, done);
1975     __ csinc(dst->as_register(), zr, zr, Assembler::EQ);
1976     __ bind(done);
1977   } else {
1978     ShouldNotReachHere();
1979   }
1980 }
1981 
1982 
1983 void LIR_Assembler::align_call(LIR_Code code) {  }
1984 
1985 
1986 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
1987   address call = __ trampoline_call(Address(op->addr(), rtype));
1988   if (call == NULL) {
1989     bailout("trampoline stub overflow");
1990     return;
1991   }
1992   add_call_info(code_offset(), op->info());
1993 }
1994 
1995 
1996 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
1997   address call = __ ic_call(op->addr());
1998   if (call == NULL) {
1999     bailout("trampoline stub overflow");
2000     return;
2001   }
2002   add_call_info(code_offset(), op->info());
2003 }
2004 
2005 
2006 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2007 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2008   ShouldNotReachHere();
2009 }
2010 
2011 
2012 void LIR_Assembler::emit_static_call_stub() {
2013   address call_pc = __ pc();
2014   address stub = __ start_a_stub(call_stub_size());
2015   if (stub == NULL) {
2016     bailout("static call stub overflow");
2017     return;
2018   }
2019 
2020   int start = __ offset();
2021 
2022   __ relocate(static_stub_Relocation::spec(call_pc));
2023   __ emit_static_call_stub();
2024 
2025   assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size()
2026         <= call_stub_size(), "stub too big");
2027   __ end_a_stub();
2028 }
2029 
2030 
2031 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2032   assert(exceptionOop->as_register() == r0, "must match");
2033   assert(exceptionPC->as_register() == r3, "must match");
2034 
2035   // exception object is not added to oop map by LinearScan
2036   // (LinearScan assumes that no oops are in fixed registers)
2037   info->add_register_oop(exceptionOop);
2038   Runtime1::StubID unwind_id;
2039 
2040   // get current pc information
2041   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2042   int pc_for_athrow_offset = __ offset();
2043   InternalAddress pc_for_athrow(__ pc());
2044   __ adr(exceptionPC->as_register(), pc_for_athrow);
2045   add_call_info(pc_for_athrow_offset, info); // for exception handler
2046 
2047   __ verify_not_null_oop(r0);
2048   // search an exception handler (r0: exception oop, r3: throwing pc)
2049   if (compilation()->has_fpu_code()) {
2050     unwind_id = Runtime1::handle_exception_id;
2051   } else {
2052     unwind_id = Runtime1::handle_exception_nofpu_id;
2053   }
2054   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2055 
2056   // FIXME: enough room for two byte trap   ????
2057   __ nop();
2058 }
2059 
2060 
2061 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2062   assert(exceptionOop->as_register() == r0, "must match");
2063 
2064   __ b(_unwind_handler_entry);
2065 }
2066 
2067 
2068 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2069   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2070   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2071 
2072   switch (left->type()) {
2073     case T_INT: {
2074       switch (code) {
2075       case lir_shl:  __ lslvw (dreg, lreg, count->as_register()); break;
2076       case lir_shr:  __ asrvw (dreg, lreg, count->as_register()); break;
2077       case lir_ushr: __ lsrvw (dreg, lreg, count->as_register()); break;
2078       default:
2079         ShouldNotReachHere();
2080         break;
2081       }
2082       break;
2083     case T_LONG:
2084     case T_ADDRESS:
2085     case T_OBJECT:
2086       switch (code) {
2087       case lir_shl:  __ lslv (dreg, lreg, count->as_register()); break;
2088       case lir_shr:  __ asrv (dreg, lreg, count->as_register()); break;
2089       case lir_ushr: __ lsrv (dreg, lreg, count->as_register()); break;
2090       default:
2091         ShouldNotReachHere();
2092         break;
2093       }
2094       break;
2095     default:
2096       ShouldNotReachHere();
2097       break;
2098     }
2099   }
2100 }
2101 
2102 
2103 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2104   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2105   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2106 
2107   switch (left->type()) {
2108     case T_INT: {
2109       switch (code) {
2110       case lir_shl:  __ lslw (dreg, lreg, count); break;
2111       case lir_shr:  __ asrw (dreg, lreg, count); break;
2112       case lir_ushr: __ lsrw (dreg, lreg, count); break;
2113       default:
2114         ShouldNotReachHere();
2115         break;
2116       }
2117       break;
2118     case T_LONG:
2119     case T_ADDRESS:
2120     case T_OBJECT:
2121       switch (code) {
2122       case lir_shl:  __ lsl (dreg, lreg, count); break;
2123       case lir_shr:  __ asr (dreg, lreg, count); break;
2124       case lir_ushr: __ lsr (dreg, lreg, count); break;
2125       default:
2126         ShouldNotReachHere();
2127         break;
2128       }
2129       break;
2130     default:
2131       ShouldNotReachHere();
2132       break;
2133     }
2134   }
2135 }
2136 
2137 
2138 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) {
2139   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2140   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2141   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2142   __ str (r, Address(sp, offset_from_rsp_in_bytes));
2143 }
2144 
2145 
2146 void LIR_Assembler::store_parameter(jint c,     int offset_from_rsp_in_words) {
2147   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2148   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2149   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2150   __ mov (rscratch1, c);
2151   __ str (rscratch1, Address(sp, offset_from_rsp_in_bytes));
2152 }
2153 
2154 
2155 void LIR_Assembler::store_parameter(jobject o,  int offset_from_rsp_in_words) {
2156   ShouldNotReachHere();
2157   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2158   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2159   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2160   __ lea(rscratch1, __ constant_oop_address(o));
2161   __ str(rscratch1, Address(sp, offset_from_rsp_in_bytes));
2162 }
2163 
2164 
2165 // This code replaces a call to arraycopy; no exception may
2166 // be thrown in this code, they must be thrown in the System.arraycopy
2167 // activation frame; we could save some checks if this would not be the case
2168 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2169   ciArrayKlass* default_type = op->expected_type();
2170   Register src = op->src()->as_register();
2171   Register dst = op->dst()->as_register();
2172   Register src_pos = op->src_pos()->as_register();
2173   Register dst_pos = op->dst_pos()->as_register();
2174   Register length  = op->length()->as_register();
2175   Register tmp = op->tmp()->as_register();
2176 
2177   CodeStub* stub = op->stub();
2178   int flags = op->flags();
2179   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2180   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2181 
2182   // if we don't know anything, just go through the generic arraycopy
2183   if (default_type == NULL // || basic_type == T_OBJECT
2184       ) {
2185     Label done;
2186     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2187 
2188     // Save the arguments in case the generic arraycopy fails and we
2189     // have to fall back to the JNI stub
2190     __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2191     __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2192     __ str(src,              Address(sp, 4*BytesPerWord));
2193 
2194     address copyfunc_addr = StubRoutines::generic_arraycopy();
2195     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
2196 
2197     // The arguments are in java calling convention so we shift them
2198     // to C convention
2199     assert_different_registers(c_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4);
2200     __ mov(c_rarg0, j_rarg0);
2201     assert_different_registers(c_rarg1, j_rarg2, j_rarg3, j_rarg4);
2202     __ mov(c_rarg1, j_rarg1);
2203     assert_different_registers(c_rarg2, j_rarg3, j_rarg4);
2204     __ mov(c_rarg2, j_rarg2);
2205     assert_different_registers(c_rarg3, j_rarg4);
2206     __ mov(c_rarg3, j_rarg3);
2207     __ mov(c_rarg4, j_rarg4);
2208 #ifndef PRODUCT
2209     if (PrintC1Statistics) {
2210       __ incrementw(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2211     }
2212 #endif
2213     __ far_call(RuntimeAddress(copyfunc_addr));
2214 
2215     __ cbz(r0, *stub->continuation());
2216 
2217     // Reload values from the stack so they are where the stub
2218     // expects them.
2219     __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2220     __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2221     __ ldr(src,              Address(sp, 4*BytesPerWord));
2222 
2223     // r0 is -1^K where K == partial copied count
2224     __ eonw(rscratch1, r0, zr);
2225     // adjust length down and src/end pos up by partial copied count
2226     __ subw(length, length, rscratch1);
2227     __ addw(src_pos, src_pos, rscratch1);
2228     __ addw(dst_pos, dst_pos, rscratch1);
2229     __ b(*stub->entry());
2230 
2231     __ bind(*stub->continuation());
2232     return;
2233   }
2234 
2235   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2236 
2237   int elem_size = type2aelembytes(basic_type);
2238   int shift_amount;
2239   int scale = exact_log2(elem_size);
2240 
2241   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2242   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2243   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2244   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2245 
2246   // test for NULL
2247   if (flags & LIR_OpArrayCopy::src_null_check) {
2248     __ cbz(src, *stub->entry());
2249   }
2250   if (flags & LIR_OpArrayCopy::dst_null_check) {
2251     __ cbz(dst, *stub->entry());
2252   }
2253 
2254   // If the compiler was not able to prove that exact type of the source or the destination
2255   // of the arraycopy is an array type, check at runtime if the source or the destination is
2256   // an instance type.
2257   if (flags & LIR_OpArrayCopy::type_check) {
2258     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2259       __ load_klass(tmp, dst);
2260       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2261       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2262       __ br(Assembler::GE, *stub->entry());
2263     }
2264 
2265     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2266       __ load_klass(tmp, src);
2267       __ ldrw(rscratch1, Address(tmp, in_bytes(Klass::layout_helper_offset())));
2268       __ cmpw(rscratch1, Klass::_lh_neutral_value);
2269       __ br(Assembler::GE, *stub->entry());
2270     }
2271   }
2272 
2273   // check if negative
2274   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2275     __ cmpw(src_pos, 0);
2276     __ br(Assembler::LT, *stub->entry());
2277   }
2278   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2279     __ cmpw(dst_pos, 0);
2280     __ br(Assembler::LT, *stub->entry());
2281   }
2282 
2283   if (flags & LIR_OpArrayCopy::length_positive_check) {
2284     __ cmpw(length, 0);
2285     __ br(Assembler::LT, *stub->entry());
2286   }
2287 
2288   if (flags & LIR_OpArrayCopy::src_range_check) {
2289     __ addw(tmp, src_pos, length);
2290     __ ldrw(rscratch1, src_length_addr);
2291     __ cmpw(tmp, rscratch1);
2292     __ br(Assembler::HI, *stub->entry());
2293   }
2294   if (flags & LIR_OpArrayCopy::dst_range_check) {
2295     __ addw(tmp, dst_pos, length);
2296     __ ldrw(rscratch1, dst_length_addr);
2297     __ cmpw(tmp, rscratch1);
2298     __ br(Assembler::HI, *stub->entry());
2299   }
2300 
2301   if (flags & LIR_OpArrayCopy::type_check) {
2302     // We don't know the array types are compatible
2303     if (basic_type != T_OBJECT) {
2304       // Simple test for basic type arrays
2305       if (UseCompressedClassPointers) {
2306         __ ldrw(tmp, src_klass_addr);
2307         __ ldrw(rscratch1, dst_klass_addr);
2308         __ cmpw(tmp, rscratch1);
2309       } else {
2310         __ ldr(tmp, src_klass_addr);
2311         __ ldr(rscratch1, dst_klass_addr);
2312         __ cmp(tmp, rscratch1);
2313       }
2314       __ br(Assembler::NE, *stub->entry());
2315     } else {
2316       // For object arrays, if src is a sub class of dst then we can
2317       // safely do the copy.
2318       Label cont, slow;
2319 
2320 #define PUSH(r1, r2)                                    \
2321       stp(r1, r2, __ pre(sp, -2 * wordSize));
2322 
2323 #define POP(r1, r2)                                     \
2324       ldp(r1, r2, __ post(sp, 2 * wordSize));
2325 
2326       __ PUSH(src, dst);
2327 
2328       __ load_klass(src, src);
2329       __ load_klass(dst, dst);
2330 
2331       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2332 
2333       __ PUSH(src, dst);
2334       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2335       __ POP(src, dst);
2336 
2337       __ cbnz(src, cont);
2338 
2339       __ bind(slow);
2340       __ POP(src, dst);
2341 
2342       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2343       if (copyfunc_addr != NULL) { // use stub if available
2344         // src is not a sub class of dst so we have to do a
2345         // per-element check.
2346 
2347         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2348         if ((flags & mask) != mask) {
2349           // Check that at least both of them object arrays.
2350           assert(flags & mask, "one of the two should be known to be an object array");
2351 
2352           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2353             __ load_klass(tmp, src);
2354           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2355             __ load_klass(tmp, dst);
2356           }
2357           int lh_offset = in_bytes(Klass::layout_helper_offset());
2358           Address klass_lh_addr(tmp, lh_offset);
2359           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2360           __ ldrw(rscratch1, klass_lh_addr);
2361           __ mov(rscratch2, objArray_lh);
2362           __ eorw(rscratch1, rscratch1, rscratch2);
2363           __ cbnzw(rscratch1, *stub->entry());
2364         }
2365 
2366        // Spill because stubs can use any register they like and it's
2367        // easier to restore just those that we care about.
2368         __ stp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2369         __ stp(length,  src_pos, Address(sp, 2*BytesPerWord));
2370         __ str(src,              Address(sp, 4*BytesPerWord));
2371 
2372         __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2373         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2374         assert_different_registers(c_rarg0, dst, dst_pos, length);
2375         __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2376         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2377         assert_different_registers(c_rarg1, dst, length);
2378         __ uxtw(c_rarg2, length);
2379         assert_different_registers(c_rarg2, dst);
2380 
2381         __ load_klass(c_rarg4, dst);
2382         __ ldr(c_rarg4, Address(c_rarg4, ObjArrayKlass::element_klass_offset()));
2383         __ ldrw(c_rarg3, Address(c_rarg4, Klass::super_check_offset_offset()));
2384         __ far_call(RuntimeAddress(copyfunc_addr));
2385 
2386 #ifndef PRODUCT
2387         if (PrintC1Statistics) {
2388           Label failed;
2389           __ cbnz(r0, failed);
2390           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2391           __ bind(failed);
2392         }
2393 #endif
2394 
2395         __ cbz(r0, *stub->continuation());
2396 
2397 #ifndef PRODUCT
2398         if (PrintC1Statistics) {
2399           __ incrementw(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2400         }
2401 #endif
2402         assert_different_registers(dst, dst_pos, length, src_pos, src, r0, rscratch1);
2403 
2404         // Restore previously spilled arguments
2405         __ ldp(dst,     dst_pos, Address(sp, 0*BytesPerWord));
2406         __ ldp(length,  src_pos, Address(sp, 2*BytesPerWord));
2407         __ ldr(src,              Address(sp, 4*BytesPerWord));
2408 
2409         // return value is -1^K where K is partial copied count
2410         __ eonw(rscratch1, r0, zr);
2411         // adjust length down and src/end pos up by partial copied count
2412         __ subw(length, length, rscratch1);
2413         __ addw(src_pos, src_pos, rscratch1);
2414         __ addw(dst_pos, dst_pos, rscratch1);
2415       }
2416 
2417       __ b(*stub->entry());
2418 
2419       __ bind(cont);
2420       __ POP(src, dst);
2421     }
2422   }
2423 
2424 #ifdef ASSERT
2425   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2426     // Sanity check the known type with the incoming class.  For the
2427     // primitive case the types must match exactly with src.klass and
2428     // dst.klass each exactly matching the default type.  For the
2429     // object array case, if no type check is needed then either the
2430     // dst type is exactly the expected type and the src type is a
2431     // subtype which we can't check or src is the same array as dst
2432     // but not necessarily exactly of type default_type.
2433     Label known_ok, halt;
2434     __ mov_metadata(tmp, default_type->constant_encoding());
2435     if (UseCompressedClassPointers) {
2436       __ encode_klass_not_null(tmp);
2437     }
2438 
2439     if (basic_type != T_OBJECT) {
2440 
2441       if (UseCompressedClassPointers) {
2442         __ ldrw(rscratch1, dst_klass_addr);
2443         __ cmpw(tmp, rscratch1);
2444       } else {
2445         __ ldr(rscratch1, dst_klass_addr);
2446         __ cmp(tmp, rscratch1);
2447       }
2448       __ br(Assembler::NE, halt);
2449       if (UseCompressedClassPointers) {
2450         __ ldrw(rscratch1, src_klass_addr);
2451         __ cmpw(tmp, rscratch1);
2452       } else {
2453         __ ldr(rscratch1, src_klass_addr);
2454         __ cmp(tmp, rscratch1);
2455       }
2456       __ br(Assembler::EQ, known_ok);
2457     } else {
2458       if (UseCompressedClassPointers) {
2459         __ ldrw(rscratch1, dst_klass_addr);
2460         __ cmpw(tmp, rscratch1);
2461       } else {
2462         __ ldr(rscratch1, dst_klass_addr);
2463         __ cmp(tmp, rscratch1);
2464       }
2465       __ br(Assembler::EQ, known_ok);
2466       __ cmp(src, dst);
2467       __ br(Assembler::EQ, known_ok);
2468     }
2469     __ bind(halt);
2470     __ stop("incorrect type information in arraycopy");
2471     __ bind(known_ok);
2472   }
2473 #endif
2474 
2475 #ifndef PRODUCT
2476   if (PrintC1Statistics) {
2477     __ incrementw(ExternalAddress(Runtime1::arraycopy_count_address(basic_type)));
2478   }
2479 #endif
2480 
2481   __ lea(c_rarg0, Address(src, src_pos, Address::uxtw(scale)));
2482   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2483   assert_different_registers(c_rarg0, dst, dst_pos, length);
2484   __ lea(c_rarg1, Address(dst, dst_pos, Address::uxtw(scale)));
2485   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2486   assert_different_registers(c_rarg1, dst, length);
2487   __ uxtw(c_rarg2, length);
2488   assert_different_registers(c_rarg2, dst);
2489 
2490   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2491   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2492   const char *name;
2493   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2494 
2495  CodeBlob *cb = CodeCache::find_blob(entry);
2496  if (cb) {
2497    __ far_call(RuntimeAddress(entry));
2498  } else {
2499    __ call_VM_leaf(entry, 3);
2500  }
2501 
2502   __ bind(*stub->continuation());
2503 }
2504 
2505 
2506 
2507 
2508 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2509   Register obj = op->obj_opr()->as_register();  // may not be an oop
2510   Register hdr = op->hdr_opr()->as_register();
2511   Register lock = op->lock_opr()->as_register();
2512   if (!UseFastLocking) {
2513     __ b(*op->stub()->entry());
2514   } else if (op->code() == lir_lock) {
2515     Register scratch = noreg;
2516     if (UseBiasedLocking) {
2517       scratch = op->scratch_opr()->as_register();
2518     }
2519     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2520     // add debug info for NullPointerException only if one is possible
2521     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2522     if (op->info() != NULL) {
2523       add_debug_info_for_null_check(null_check_offset, op->info());
2524     }
2525     // done
2526   } else if (op->code() == lir_unlock) {
2527     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2528     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2529   } else {
2530     Unimplemented();
2531   }
2532   __ bind(*op->stub()->continuation());
2533 }
2534 
2535 
2536 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2537   ciMethod* method = op->profiled_method();
2538   int bci          = op->profiled_bci();
2539   ciMethod* callee = op->profiled_callee();
2540 
2541   // Update counter for all call types
2542   ciMethodData* md = method->method_data_or_null();
2543   assert(md != NULL, "Sanity");
2544   ciProfileData* data = md->bci_to_data(bci);
2545   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2546   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2547   Register mdo  = op->mdo()->as_register();
2548   __ mov_metadata(mdo, md->constant_encoding());
2549   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2550   // Perform additional virtual call profiling for invokevirtual and
2551   // invokeinterface bytecodes
2552   if (op->should_profile_receiver_type()) {
2553     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2554     Register recv = op->recv()->as_register();
2555     assert_different_registers(mdo, recv);
2556     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2557     ciKlass* known_klass = op->known_holder();
2558     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2559       // We know the type that will be seen at this call site; we can
2560       // statically update the MethodData* rather than needing to do
2561       // dynamic tests on the receiver type
2562 
2563       // NOTE: we should probably put a lock around this search to
2564       // avoid collisions by concurrent compilations
2565       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2566       uint i;
2567       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2568         ciKlass* receiver = vc_data->receiver(i);
2569         if (known_klass->equals(receiver)) {
2570           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2571           __ addptr(data_addr, DataLayout::counter_increment);
2572           return;
2573         }
2574       }
2575 
2576       // Receiver type not found in profile data; select an empty slot
2577 
2578       // Note that this is less efficient than it should be because it
2579       // always does a write to the receiver part of the
2580       // VirtualCallData rather than just the first time
2581       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2582         ciKlass* receiver = vc_data->receiver(i);
2583         if (receiver == NULL) {
2584           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2585           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2586           __ lea(rscratch2, recv_addr);
2587           __ str(rscratch1, Address(rscratch2));
2588           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2589           __ addptr(data_addr, DataLayout::counter_increment);
2590           return;
2591         }
2592       }
2593     } else {
2594       __ load_klass(recv, recv);
2595       Label update_done;
2596       type_profile_helper(mdo, md, data, recv, &update_done);
2597       // Receiver did not match any saved receiver and there is no empty row for it.
2598       // Increment total counter to indicate polymorphic case.
2599       __ addptr(counter_addr, DataLayout::counter_increment);
2600 
2601       __ bind(update_done);
2602     }
2603   } else {
2604     // Static call
2605     __ addptr(counter_addr, DataLayout::counter_increment);
2606   }
2607 }
2608 
2609 
2610 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2611   Unimplemented();
2612 }
2613 
2614 
2615 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2616   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2617 }
2618 
2619 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2620   assert(op->crc()->is_single_cpu(),  "crc must be register");
2621   assert(op->val()->is_single_cpu(),  "byte value must be register");
2622   assert(op->result_opr()->is_single_cpu(), "result must be register");
2623   Register crc = op->crc()->as_register();
2624   Register val = op->val()->as_register();
2625   Register res = op->result_opr()->as_register();
2626 
2627   assert_different_registers(val, crc, res);
2628   unsigned long offset;
2629   __ adrp(res, ExternalAddress(StubRoutines::crc_table_addr()), offset);
2630   if (offset) __ add(res, res, offset);
2631 
2632   __ mvnw(crc, crc); // ~crc
2633   __ update_byte_crc32(crc, val, res);
2634   __ mvnw(res, crc); // ~crc
2635 }
2636 
2637 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2638   COMMENT("emit_profile_type {");
2639   Register obj = op->obj()->as_register();
2640   Register tmp = op->tmp()->as_pointer_register();
2641   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2642   ciKlass* exact_klass = op->exact_klass();
2643   intptr_t current_klass = op->current_klass();
2644   bool not_null = op->not_null();
2645   bool no_conflict = op->no_conflict();
2646 
2647   Label update, next, none;
2648 
2649   bool do_null = !not_null;
2650   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2651   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2652 
2653   assert(do_null || do_update, "why are we here?");
2654   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2655   assert(mdo_addr.base() != rscratch1, "wrong register");
2656 
2657   __ verify_oop(obj);
2658 
2659   if (tmp != obj) {
2660     __ mov(tmp, obj);
2661   }
2662   if (do_null) {
2663     __ cbnz(tmp, update);
2664     if (!TypeEntries::was_null_seen(current_klass)) {
2665       __ ldr(rscratch2, mdo_addr);
2666       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2667       __ str(rscratch2, mdo_addr);
2668     }
2669     if (do_update) {
2670 #ifndef ASSERT
2671       __ b(next);
2672     }
2673 #else
2674       __ b(next);
2675     }
2676   } else {
2677     __ cbnz(tmp, update);
2678     __ stop("unexpected null obj");
2679 #endif
2680   }
2681 
2682   __ bind(update);
2683 
2684   if (do_update) {
2685 #ifdef ASSERT
2686     if (exact_klass != NULL) {
2687       Label ok;
2688       __ load_klass(tmp, tmp);
2689       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2690       __ eor(rscratch1, tmp, rscratch1);
2691       __ cbz(rscratch1, ok);
2692       __ stop("exact klass and actual klass differ");
2693       __ bind(ok);
2694     }
2695 #endif
2696     if (!no_conflict) {
2697       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2698         if (exact_klass != NULL) {
2699           __ mov_metadata(tmp, exact_klass->constant_encoding());
2700         } else {
2701           __ load_klass(tmp, tmp);
2702         }
2703 
2704         __ ldr(rscratch2, mdo_addr);
2705         __ eor(tmp, tmp, rscratch2);
2706         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2707         // klass seen before, nothing to do. The unknown bit may have been
2708         // set already but no need to check.
2709         __ cbz(rscratch1, next);
2710 
2711         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2712 
2713         if (TypeEntries::is_type_none(current_klass)) {
2714           __ cbz(rscratch2, none);
2715           __ cmp(rscratch2, TypeEntries::null_seen);
2716           __ br(Assembler::EQ, none);
2717           // There is a chance that the checks above (re-reading profiling
2718           // data from memory) fail if another thread has just set the
2719           // profiling to this obj's klass
2720           __ dmb(Assembler::ISHLD);
2721           __ ldr(rscratch2, mdo_addr);
2722           __ eor(tmp, tmp, rscratch2);
2723           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2724           __ cbz(rscratch1, next);
2725         }
2726       } else {
2727         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2728                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2729 
2730         __ ldr(tmp, mdo_addr);
2731         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2732       }
2733 
2734       // different than before. Cannot keep accurate profile.
2735       __ ldr(rscratch2, mdo_addr);
2736       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2737       __ str(rscratch2, mdo_addr);
2738 
2739       if (TypeEntries::is_type_none(current_klass)) {
2740         __ b(next);
2741 
2742         __ bind(none);
2743         // first time here. Set profile type.
2744         __ str(tmp, mdo_addr);
2745       }
2746     } else {
2747       // There's a single possible klass at this profile point
2748       assert(exact_klass != NULL, "should be");
2749       if (TypeEntries::is_type_none(current_klass)) {
2750         __ mov_metadata(tmp, exact_klass->constant_encoding());
2751         __ ldr(rscratch2, mdo_addr);
2752         __ eor(tmp, tmp, rscratch2);
2753         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2754         __ cbz(rscratch1, next);
2755 #ifdef ASSERT
2756         {
2757           Label ok;
2758           __ ldr(rscratch1, mdo_addr);
2759           __ cbz(rscratch1, ok);
2760           __ cmp(rscratch1, TypeEntries::null_seen);
2761           __ br(Assembler::EQ, ok);
2762           // may have been set by another thread
2763           __ dmb(Assembler::ISHLD);
2764           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2765           __ ldr(rscratch2, mdo_addr);
2766           __ eor(rscratch2, rscratch1, rscratch2);
2767           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2768           __ cbz(rscratch2, ok);
2769 
2770           __ stop("unexpected profiling mismatch");
2771           __ bind(ok);
2772         }
2773 #endif
2774         // first time here. Set profile type.
2775         __ ldr(tmp, mdo_addr);
2776       } else {
2777         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2778                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2779 
2780         __ ldr(tmp, mdo_addr);
2781         __ tbnz(tmp, exact_log2(TypeEntries::type_unknown), next); // already unknown. Nothing to do anymore.
2782 
2783         __ orr(tmp, tmp, TypeEntries::type_unknown);
2784         __ str(tmp, mdo_addr);
2785         // FIXME: Write barrier needed here?
2786       }
2787     }
2788 
2789     __ bind(next);
2790   }
2791   COMMENT("} emit_profile_type");
2792 }
2793 
2794 
2795 void LIR_Assembler::align_backward_branch_target() {
2796 }
2797 
2798 
2799 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
2800   // tmp must be unused
2801   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
2802 
2803   if (left->is_single_cpu()) {
2804     assert(dest->is_single_cpu(), "expect single result reg");
2805     __ negw(dest->as_register(), left->as_register());
2806   } else if (left->is_double_cpu()) {
2807     assert(dest->is_double_cpu(), "expect double result reg");
2808     __ neg(dest->as_register_lo(), left->as_register_lo());
2809   } else if (left->is_single_fpu()) {
2810     assert(dest->is_single_fpu(), "expect single float result reg");
2811     __ fnegs(dest->as_float_reg(), left->as_float_reg());
2812   } else {
2813     assert(left->is_double_fpu(), "expect double float operand reg");
2814     assert(dest->is_double_fpu(), "expect double float result reg");
2815     __ fnegd(dest->as_double_reg(), left->as_double_reg());
2816   }
2817 }
2818 
2819 
2820 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
2821   assert(patch_code == lir_patch_none, "Patch code not supported");
2822   __ lea(dest->as_register_lo(), as_Address(addr->as_address_ptr()));
2823 }
2824 
2825 
2826 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2827   assert(!tmp->is_valid(), "don't need temporary");
2828 
2829   CodeBlob *cb = CodeCache::find_blob(dest);
2830   if (cb) {
2831     __ far_call(RuntimeAddress(dest));
2832   } else {
2833     __ mov(rscratch1, RuntimeAddress(dest));
2834     __ blr(rscratch1);
2835   }
2836 
2837   if (info != NULL) {
2838     add_call_info_here(info);
2839   }
2840   __ maybe_isb();
2841 }
2842 
2843 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2844   if (dest->is_address() || src->is_address()) {
2845     move_op(src, dest, type, lir_patch_none, info,
2846             /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2847   } else {
2848     ShouldNotReachHere();
2849   }
2850 }
2851 
2852 #ifdef ASSERT
2853 // emit run-time assertion
2854 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2855   assert(op->code() == lir_assert, "must be");
2856 
2857   if (op->in_opr1()->is_valid()) {
2858     assert(op->in_opr2()->is_valid(), "both operands must be valid");
2859     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2860   } else {
2861     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2862     assert(op->condition() == lir_cond_always, "no other conditions allowed");
2863   }
2864 
2865   Label ok;
2866   if (op->condition() != lir_cond_always) {
2867     Assembler::Condition acond = Assembler::AL;
2868     switch (op->condition()) {
2869       case lir_cond_equal:        acond = Assembler::EQ;  break;
2870       case lir_cond_notEqual:     acond = Assembler::NE;  break;
2871       case lir_cond_less:         acond = Assembler::LT;  break;
2872       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
2873       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
2874       case lir_cond_greater:      acond = Assembler::GT;  break;
2875       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
2876       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
2877       default:                    ShouldNotReachHere();
2878     }
2879     __ br(acond, ok);
2880   }
2881   if (op->halt()) {
2882     const char* str = __ code_string(op->msg());
2883     __ stop(str);
2884   } else {
2885     breakpoint();
2886   }
2887   __ bind(ok);
2888 }
2889 #endif
2890 
2891 #ifndef PRODUCT
2892 #define COMMENT(x)   do { __ block_comment(x); } while (0)
2893 #else
2894 #define COMMENT(x)
2895 #endif
2896 
2897 void LIR_Assembler::membar() {
2898   COMMENT("membar");
2899   __ membar(MacroAssembler::AnyAny);
2900 }
2901 
2902 void LIR_Assembler::membar_acquire() {
2903   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
2904 }
2905 
2906 void LIR_Assembler::membar_release() {
2907   __ membar(Assembler::LoadStore|Assembler::StoreStore);
2908 }
2909 
2910 void LIR_Assembler::membar_loadload() {
2911   __ membar(Assembler::LoadLoad);
2912 }
2913 
2914 void LIR_Assembler::membar_storestore() {
2915   __ membar(MacroAssembler::StoreStore);
2916 }
2917 
2918 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
2919 
2920 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
2921 
2922 void LIR_Assembler::on_spin_wait() {
2923   Unimplemented();
2924 }
2925 
2926 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
2927   __ mov(result_reg->as_register(), rthread);
2928 }
2929 
2930 
2931 void LIR_Assembler::peephole(LIR_List *lir) {
2932 #if 0
2933   if (tableswitch_count >= max_tableswitches)
2934     return;
2935 
2936   /*
2937     This finite-state automaton recognizes sequences of compare-and-
2938     branch instructions.  We will turn them into a tableswitch.  You
2939     could argue that C1 really shouldn't be doing this sort of
2940     optimization, but without it the code is really horrible.
2941   */
2942 
2943   enum { start_s, cmp1_s, beq_s, cmp_s } state;
2944   int first_key, last_key = -2147483648;
2945   int next_key = 0;
2946   int start_insn = -1;
2947   int last_insn = -1;
2948   Register reg = noreg;
2949   LIR_Opr reg_opr;
2950   state = start_s;
2951 
2952   LIR_OpList* inst = lir->instructions_list();
2953   for (int i = 0; i < inst->length(); i++) {
2954     LIR_Op* op = inst->at(i);
2955     switch (state) {
2956     case start_s:
2957       first_key = -1;
2958       start_insn = i;
2959       switch (op->code()) {
2960       case lir_cmp:
2961         LIR_Opr opr1 = op->as_Op2()->in_opr1();
2962         LIR_Opr opr2 = op->as_Op2()->in_opr2();
2963         if (opr1->is_cpu_register() && opr1->is_single_cpu()
2964             && opr2->is_constant()
2965             && opr2->type() == T_INT) {
2966           reg_opr = opr1;
2967           reg = opr1->as_register();
2968           first_key = opr2->as_constant_ptr()->as_jint();
2969           next_key = first_key + 1;
2970           state = cmp_s;
2971           goto next_state;
2972         }
2973         break;
2974       }
2975       break;
2976     case cmp_s:
2977       switch (op->code()) {
2978       case lir_branch:
2979         if (op->as_OpBranch()->cond() == lir_cond_equal) {
2980           state = beq_s;
2981           last_insn = i;
2982           goto next_state;
2983         }
2984       }
2985       state = start_s;
2986       break;
2987     case beq_s:
2988       switch (op->code()) {
2989       case lir_cmp: {
2990         LIR_Opr opr1 = op->as_Op2()->in_opr1();
2991         LIR_Opr opr2 = op->as_Op2()->in_opr2();
2992         if (opr1->is_cpu_register() && opr1->is_single_cpu()
2993             && opr1->as_register() == reg
2994             && opr2->is_constant()
2995             && opr2->type() == T_INT
2996             && opr2->as_constant_ptr()->as_jint() == next_key) {
2997           last_key = next_key;
2998           next_key++;
2999           state = cmp_s;
3000           goto next_state;
3001         }
3002       }
3003       }
3004       last_key = next_key;
3005       state = start_s;
3006       break;
3007     default:
3008       assert(false, "impossible state");
3009     }
3010     if (state == start_s) {
3011       if (first_key < last_key - 5L && reg != noreg) {
3012         {
3013           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3014           //        reg->encoding(),
3015           //        start_insn, first_key, last_key);
3016           //   for (int i = 0; i < inst->length(); i++) {
3017           //     inst->at(i)->print();
3018           //     tty->print("\n");
3019           //   }
3020           //   tty->print("\n");
3021         }
3022 
3023         struct tableswitch *sw = &switches[tableswitch_count];
3024         sw->_insn_index = start_insn, sw->_first_key = first_key,
3025           sw->_last_key = last_key, sw->_reg = reg;
3026         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3027         {
3028           // Insert the new table of branches
3029           int offset = last_insn;
3030           for (int n = first_key; n < last_key; n++) {
3031             inst->insert_before
3032               (last_insn + 1,
3033                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3034                                 inst->at(offset)->as_OpBranch()->label()));
3035             offset -= 2, i++;
3036           }
3037         }
3038         // Delete all the old compare-and-branch instructions
3039         for (int n = first_key; n < last_key; n++) {
3040           inst->remove_at(start_insn);
3041           inst->remove_at(start_insn);
3042         }
3043         // Insert the tableswitch instruction
3044         inst->insert_before(start_insn,
3045                             new LIR_Op2(lir_cmp, lir_cond_always,
3046                                         LIR_OprFact::intConst(tableswitch_count),
3047                                         reg_opr));
3048         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3049         tableswitch_count++;
3050       }
3051       reg = noreg;
3052       last_key = -2147483648;
3053     }
3054   next_state:
3055     ;
3056   }
3057 #endif
3058 }
3059 
3060 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3061   Address addr = as_Address(src->as_address_ptr());
3062   BasicType type = src->type();
3063   bool is_oop = type == T_OBJECT || type == T_ARRAY;
3064 
3065   void (MacroAssembler::* add)(Register prev, RegisterOrConstant incr, Register addr);
3066   void (MacroAssembler::* xchg)(Register prev, Register newv, Register addr);
3067 
3068   switch(type) {
3069   case T_INT:
3070     xchg = &MacroAssembler::atomic_xchgalw;
3071     add = &MacroAssembler::atomic_addalw;
3072     break;
3073   case T_LONG:
3074     xchg = &MacroAssembler::atomic_xchgal;
3075     add = &MacroAssembler::atomic_addal;
3076     break;
3077   case T_OBJECT:
3078   case T_ARRAY:
3079     if (UseCompressedOops) {
3080       xchg = &MacroAssembler::atomic_xchgalw;
3081       add = &MacroAssembler::atomic_addalw;
3082     } else {
3083       xchg = &MacroAssembler::atomic_xchgal;
3084       add = &MacroAssembler::atomic_addal;
3085     }
3086     break;
3087   default:
3088     ShouldNotReachHere();
3089     xchg = &MacroAssembler::atomic_xchgal;
3090     add = &MacroAssembler::atomic_addal; // unreachable
3091   }
3092 
3093   switch (code) {
3094   case lir_xadd:
3095     {
3096       RegisterOrConstant inc;
3097       Register tmp = as_reg(tmp_op);
3098       Register dst = as_reg(dest);
3099       if (data->is_constant()) {
3100         inc = RegisterOrConstant(as_long(data));
3101         assert_different_registers(dst, addr.base(), tmp,
3102                                    rscratch1, rscratch2);
3103       } else {
3104         inc = RegisterOrConstant(as_reg(data));
3105         assert_different_registers(inc.as_register(), dst, addr.base(), tmp,
3106                                    rscratch1, rscratch2);
3107       }
3108       __ lea(tmp, addr);
3109       (_masm->*add)(dst, inc, tmp);
3110       break;
3111     }
3112   case lir_xchg:
3113     {
3114       Register tmp = tmp_op->as_register();
3115       Register obj = as_reg(data);
3116       Register dst = as_reg(dest);
3117       if (is_oop && UseCompressedOops) {
3118         __ encode_heap_oop(rscratch2, obj);
3119         obj = rscratch2;
3120       }
3121       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3122       __ lea(tmp, addr);
3123       (_masm->*xchg)(dst, obj, tmp);
3124       if (is_oop && UseCompressedOops) {
3125         __ decode_heap_oop(dst);
3126       }
3127     }
3128     break;
3129   default:
3130     ShouldNotReachHere();
3131   }
3132   __ membar(__ AnyAny);
3133 }
3134 
3135 #undef __