1 /*
   2  * Copyright (c) 2000, 2015, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This code is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 only, as
   8  * published by the Free Software Foundation.
   9  *
  10  * This code is distributed in the hope that it will be useful, but WITHOUT
  11  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 // This file is a derivative work resulting from (and including) modifications
  26 // made by Azul Systems, Inc.  The dates of such changes are 2013-2016.
  27 // Copyright 2013-2016 Azul Systems, Inc.  All Rights Reserved.
  28 //
  29 // Please contact Azul Systems, 385 Moffett Park Drive, Suite 115, Sunnyvale,
  30 // CA 94089 USA or visit www.azul.com if you need additional information or
  31 // have any questions.
  32 
  33 #include "precompiled.hpp"
  34 #include "asm/assembler.hpp"
  35 #include "c1/c1_CodeStubs.hpp"
  36 #include "c1/c1_Compilation.hpp"
  37 #include "c1/c1_LIRAssembler.hpp"
  38 #include "c1/c1_MacroAssembler.hpp"
  39 #include "c1/c1_Runtime1.hpp"
  40 #include "c1/c1_ValueStack.hpp"
  41 #include "ci/ciArrayKlass.hpp"
  42 #include "ci/ciInstance.hpp"
  43 #include "gc_interface/collectedHeap.hpp"
  44 #include "memory/barrierSet.hpp"
  45 #include "memory/cardTableModRefBS.hpp"
  46 #include "nativeInst_aarch32.hpp"
  47 #include "oops/objArrayKlass.hpp"
  48 #include "runtime/sharedRuntime.hpp"
  49 #include "vmreg_aarch32.inline.hpp"
  50 
  51 #ifndef PRODUCT
  52 #define COMMENT(x)   do { __ block_comment(x); } while (0)
  53 #else
  54 #define COMMENT(x)
  55 #endif
  56 
  57 NEEDS_CLEANUP // remove this definitions ?
  58 const Register IC_Klass    = rscratch2;   // where the IC klass is cached
  59 const Register SYNC_header = r0;   // synchronization header
  60 const Register SHIFT_count = r0;   // where count for shift operations must be
  61 
  62 #define __ _masm->
  63 
  64 
  65 static void select_different_registers(Register preserve,
  66                                        Register extra,
  67                                        Register &tmp1,
  68                                        Register &tmp2) {
  69   if (tmp1 == preserve) {
  70     assert_different_registers(tmp1, tmp2, extra);
  71     tmp1 = extra;
  72   } else if (tmp2 == preserve) {
  73     assert_different_registers(tmp1, tmp2, extra);
  74     tmp2 = extra;
  75   }
  76   assert_different_registers(preserve, tmp1, tmp2);
  77 }
  78 
  79 
  80 
  81 static void select_different_registers(Register preserve,
  82                                        Register extra,
  83                                        Register &tmp1,
  84                                        Register &tmp2,
  85                                        Register &tmp3) {
  86   if (tmp1 == preserve) {
  87     assert_different_registers(tmp1, tmp2, tmp3, extra);
  88     tmp1 = extra;
  89   } else if (tmp2 == preserve) {
  90     assert_different_registers(tmp1, tmp2, tmp3, extra);
  91     tmp2 = extra;
  92   } else if (tmp3 == preserve) {
  93     assert_different_registers(tmp1, tmp2, tmp3, extra);
  94     tmp3 = extra;
  95   }
  96   assert_different_registers(preserve, tmp1, tmp2, tmp3);
  97 }
  98 
  99 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; }
 100 
 101 
 102 LIR_Opr LIR_Assembler::receiverOpr() {
 103   return FrameMap::receiver_opr;
 104 }
 105 
 106 LIR_Opr LIR_Assembler::osrBufferPointer() {
 107   return FrameMap::as_pointer_opr(receiverOpr()->as_register());
 108 }
 109 
 110 //--------------fpu register translations-----------------------
 111 
 112 
 113 address LIR_Assembler::float_constant(float f) {
 114   address const_addr = __ float_constant(f);
 115   if (const_addr == NULL) {
 116     bailout("const section overflow");
 117     return __ code()->consts()->start();
 118   } else {
 119     return const_addr;
 120   }
 121 }
 122 
 123 
 124 address LIR_Assembler::double_constant(double d) {
 125   address const_addr = __ double_constant(d);
 126   if (const_addr == NULL) {
 127     bailout("const section overflow");
 128     return __ code()->consts()->start();
 129   } else {
 130     return const_addr;
 131   }
 132 }
 133 
 134 void LIR_Assembler::set_24bit_FPU() { Unimplemented(); }
 135 
 136 void LIR_Assembler::reset_FPU() { Unimplemented(); }
 137 
 138 void LIR_Assembler::fpop() { Unimplemented(); }
 139 
 140 void LIR_Assembler::fxch(int i) { Unimplemented(); }
 141 
 142 void LIR_Assembler::fld(int i) { Unimplemented(); }
 143 
 144 void LIR_Assembler::ffree(int i) { Unimplemented(); }
 145 
 146 void LIR_Assembler::breakpoint() { __ bkpt(0); }
 147 
 148 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); }
 149 
 150 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); }
 151 
 152 //-------------------------------------------
 153 
 154 static Register as_reg(LIR_Opr op) {
 155   return op->is_double_cpu() ? op->as_register_lo() : op->as_register();
 156 }
 157 
 158 Address LIR_Assembler::as_Address(LIR_Address* addr) {
 159   // as_Address(LIR_Address*, Address::InsnDataType) should be used instead
 160   ShouldNotCallThis();
 161 }
 162 
 163 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
 164   // as_Address_hi(LIR_Address*, Address::InsnDataType) should be used instead
 165   ShouldNotCallThis();
 166 }
 167 
 168 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
 169   // as_Address_lo(LIR_Address*, Address::InsnDataType) should be used instead
 170   ShouldNotCallThis();
 171 }
 172 
 173 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp, Address::InsnDataType type) {
 174   if (addr->base()->is_illegal()) {
 175     assert(addr->index()->is_illegal(), "must be illegal too");
 176     __ mov(tmp, addr->disp());
 177     return Address(tmp); // encoding is ok for any data type
 178   }
 179 
 180   Register base = addr->base()->as_pointer_register();
 181 
 182   if (addr->index()->is_illegal()) {
 183     return Address(base, addr->disp()).safe_for(type, _masm, tmp);
 184   } else if (addr->index()->is_cpu_register()) {
 185     assert(addr->disp() == 0, "must be");
 186     Register index = addr->index()->as_pointer_register();
 187     return Address(base, index, lsl(addr->scale())).safe_for(type, _masm, tmp);
 188   } else if (addr->index()->is_constant()) {
 189     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp();
 190     return Address(base, addr_offset).safe_for(type, _masm, tmp);
 191   }
 192 
 193   Unimplemented();
 194   return Address();
 195 }
 196 
 197 Address LIR_Assembler::as_Address_hi(LIR_Address* addr, Address::InsnDataType type) {
 198   assert(type == Address::IDT_INT, "only to be used for accessing high word of jlong");
 199 
 200   if (addr->base()->is_illegal()) {
 201     assert(addr->index()->is_illegal(), "must be illegal too");
 202     __ mov(rscratch1, addr->disp() + wordSize);
 203     return Address(rscratch1); // encoding is ok for IDR_INT
 204   }
 205 
 206   Register base = addr->base()->as_pointer_register();
 207 
 208   if (addr->index()->is_illegal()) {
 209     return Address(base, addr->disp() + wordSize).safe_for(Address::IDT_INT, _masm, rscratch1);
 210   } else if (addr->index()->is_cpu_register()) {
 211     assert(addr->disp() == 0, "must be");
 212     Register index = addr->index()->as_pointer_register();
 213     __ add(rscratch1, base, wordSize);
 214     return Address(rscratch1, index, lsl(addr->scale())); // encoding is ok for IDT_INT
 215   } else if (addr->index()->is_constant()) {
 216     intptr_t addr_offset = (addr->index()->as_constant_ptr()->as_jint() << addr->scale()) + addr->disp() + wordSize;
 217     return Address(base, addr_offset).safe_for(Address::IDT_INT, _masm, rscratch1);
 218   }
 219 
 220   Unimplemented();
 221   return Address();
 222 }
 223 
 224 Address LIR_Assembler::as_Address_lo(LIR_Address* addr, Address::InsnDataType type) {
 225   return as_Address(addr, rscratch1, type);
 226 }
 227 
 228 
 229 void LIR_Assembler::osr_entry() {
 230   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 231   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 232   ValueStack* entry_state = osr_entry->state();
 233   int number_of_locks = entry_state->locks_size();
 234 
 235   // we jump here if osr happens with the interpreter
 236   // state set up to continue at the beginning of the
 237   // loop that triggered osr - in particular, we have
 238   // the following registers setup:
 239   //
 240   // r1: osr buffer
 241   //
 242 
 243   // build frame
 244   ciMethod* m = compilation()->method();
 245   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 246 
 247   // OSR buffer is
 248   //
 249   // locals[nlocals-1..0]
 250   // monitors[0..number_of_locks]
 251   //
 252   // locals is a direct copy of the interpreter frame so in the osr buffer
 253   // so first slot in the local array is the last local from the interpreter
 254   // and last slot is local[0] (receiver) from the interpreter
 255   //
 256   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 257   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 258   // in the interpreter frame (the method lock if a sync method)
 259 
 260   // Initialize monitors in the compiled activation.
 261   //   r1: pointer to osr buffer
 262   //
 263   // All other registers are dead at this point and the locals will be
 264   // copied into place by code emitted in the IR.
 265 
 266   Register OSR_buf = osrBufferPointer()->as_pointer_register();
 267   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 268     int monitor_offset = BytesPerWord * method()->max_locals() +
 269       (2 * BytesPerWord) * (number_of_locks - 1);
 270     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 271     // the OSR buffer using 2 word entries: first the lock and then
 272     // the oop.
 273     for (int i = 0; i < number_of_locks; i++) {
 274       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 275 #ifdef ASSERT
 276       // verify the interpreter's monitor has a non-null object
 277       {
 278         Label L;
 279         __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 280         __ cbnz(rscratch1, L);
 281         __ stop("locked object is NULL");
 282         __ bind(L);
 283       }
 284 #endif
 285       __ ldr(rscratch1, Address(OSR_buf, slot_offset + 0));
 286       __ str(rscratch1, frame_map()->address_for_monitor_lock(i));
 287       __ ldr(rscratch1, Address(OSR_buf, slot_offset + 1*BytesPerWord));
 288       __ str(rscratch1, frame_map()->address_for_monitor_object(i));
 289     }
 290   }
 291 }
 292 
 293 
 294 // inline cache check; done before the frame is built.
 295 int LIR_Assembler::check_icache() {
 296   Register receiver = FrameMap::receiver_opr->as_register();
 297   Register ic_klass = IC_Klass;
 298   int start_offset = __ offset();
 299   __ inline_cache_check(receiver, ic_klass);
 300 
 301   // if icache check fails, then jump to runtime routine
 302   // Note: RECEIVER must still contain the receiver!
 303   Label dont;
 304   __ b(dont, Assembler::EQ);
 305   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 306 
 307   // We align the verified entry point unless the method body
 308   // (including its inline cache check) will fit in a single 64-byte
 309   // icache line.
 310   if (! method()->is_accessor() || __ offset() - start_offset > 4 * 4) {
 311     // force alignment after the cache check.
 312     __ align(CodeEntryAlignment);
 313   }
 314 
 315   __ bind(dont);
 316   return start_offset;
 317 }
 318 
 319 
 320 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 321   if (o == NULL) {
 322     __ mov(reg, 0);
 323   } else {
 324     __ movoop(reg, o, /*immediate*/true);
 325   }
 326 }
 327 
 328 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) {
 329   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::deoptimize_id)));
 330   add_call_info_here(info);
 331 }
 332 
 333 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 334   PatchingStub* patch = new PatchingStub(_masm, patching_id(info));
 335   __ relocate(oop_Relocation::spec(__ oop_recorder()->allocate_oop_index(NULL)));
 336   __ patchable_load(reg, pc());
 337   patching_epilog(patch, lir_patch_normal, reg, info);
 338 }
 339 
 340 // Return sp decrement needed to build a frame
 341 int LIR_Assembler::initial_frame_size_in_bytes() const {
 342   // We need to subtract two words to take into account saved lr and rfp.
 343   return in_bytes(frame_map()->framesize_in_bytes()) -
 344          FrameMap::frame_pad_in_bytes;
 345 }
 346 
 347 int LIR_Assembler::emit_exception_handler() {
 348   // if the last instruction is a call (typically to do a throw which
 349   // is coming at the end after block reordering) the return address
 350   // must still point into the code area in order to avoid assertion
 351   // failures when searching for the corresponding bci => add a nop
 352   // (was bug 5/14/1999 - gri)
 353   __ nop();
 354 
 355   // generate code for exception handler
 356   address handler_base = __ start_a_stub(exception_handler_size);
 357   if (handler_base == NULL) {
 358     // not enough space left for the handler
 359     bailout("exception handler overflow");
 360     return -1;
 361   }
 362 
 363   int offset = code_offset();
 364 
 365   // the exception oop and pc are in r0, and r3
 366   // no other registers need to be preserved, so invalidate them
 367   __ invalidate_registers(false, true, false);
 368 
 369   // check that there is really an exception
 370   __ verify_not_null_oop(r0);
 371 
 372   // search an exception handler (r0: exception oop, r3: throwing pc)
 373   __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id)));  __ should_not_reach_here();
 374   guarantee(code_offset() - offset <= exception_handler_size, "overflow");
 375   __ end_a_stub();
 376 
 377   return offset;
 378 }
 379 
 380 
 381 // Emit the code to remove the frame from the stack in the exception
 382 // unwind path.
 383 int LIR_Assembler::emit_unwind_handler() {
 384 #ifndef PRODUCT
 385   if (CommentedAssembly) {
 386     _masm->block_comment("Unwind handler");
 387   }
 388 #endif
 389 
 390   int offset = code_offset();
 391 
 392   // Fetch the exception from TLS and clear out exception related thread state
 393   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
 394   __ mov(rscratch1, 0);
 395   __ str(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 396   __ str(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 397 
 398   __ bind(_unwind_handler_entry);
 399   __ verify_not_null_oop(r0);
 400 
 401   // Preform needed unlocking
 402   MonitorExitStub* stub = NULL;
 403   if (method()->is_synchronized()) {
 404     monitor_address(0, FrameMap::r1_opr);
 405     stub = new MonitorExitStub(FrameMap::r1_opr, true, 0);
 406     __ unlock_object(r5, r4, r1, *stub->entry());
 407     __ bind(*stub->continuation());
 408   }
 409 
 410   if (compilation()->env()->dtrace_method_probes()) {
 411     __ call_Unimplemented();
 412 #if 0
 413     // FIXME check exception_store is not clobbered below!
 414     __ movptr(Address(rsp, 0), rax);
 415     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding());
 416     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 417 #endif
 418   }
 419 
 420   // remove the activation and dispatch to the unwind handler
 421   __ block_comment("remove_frame and dispatch to the unwind handler");
 422   __ remove_frame(initial_frame_size_in_bytes());
 423   __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id)));
 424 
 425   // Emit the slow path assembly
 426   if (stub != NULL) {
 427     stub->emit_code(this);
 428   }
 429 
 430   return offset;
 431 }
 432 
 433 
 434 int LIR_Assembler::emit_deopt_handler() {
 435   // if the last instruction is a call (typically to do a throw which
 436   // is coming at the end after block reordering) the return address
 437   // must still point into the code area in order to avoid assertion
 438   // failures when searching for the corresponding bci => add a nop
 439   // (was bug 5/14/1999 - gri)
 440   __ nop();
 441 
 442   // generate code for exception handler
 443   address handler_base = __ start_a_stub(deopt_handler_size);
 444   if (handler_base == NULL) {
 445     // not enough space left for the handler
 446     bailout("deopt handler overflow");
 447     return -1;
 448   }
 449 
 450   int offset = code_offset();
 451 
 452   __ adr(lr, pc());
 453   __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 454   guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
 455   __ end_a_stub();
 456 
 457   return offset;
 458 }
 459 
 460 
 461 // This is the fast version of java.lang.String.compare; it has not
 462 // OSR-entry and therefore, we generate a slow version for OSR's
 463 void LIR_Assembler::emit_string_compare(LIR_Opr arg0, LIR_Opr arg1, LIR_Opr dst, CodeEmitInfo* info)  {
 464   __ mov(r2, (address)__FUNCTION__);
 465   __ call_Unimplemented();
 466 }
 467 
 468 
 469 void LIR_Assembler::add_debug_info_for_branch(address adr, CodeEmitInfo* info) {
 470   _masm->code_section()->relocate(adr, relocInfo::poll_type);
 471   int pc_offset = code_offset();
 472   flush_debug_info(pc_offset);
 473   info->record_debug_info(compilation()->debug_info_recorder(), pc_offset);
 474   if (info->exception_handlers() != NULL) {
 475     compilation()->add_exception_handlers_for_pco(pc_offset, info->exception_handlers());
 476   }
 477 }
 478 
 479 void LIR_Assembler::return_op(LIR_Opr result) {
 480   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == r0, "word returns are in r0,");
 481   // Pop the stack before the safepoint code
 482   __ remove_frame(initial_frame_size_in_bytes());
 483   address polling_page(os::get_polling_page());
 484   __ read_polling_page(rscratch1, polling_page, relocInfo::poll_return_type);
 485   __ ret(lr);
 486 }
 487 
 488 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 489   address polling_page(os::get_polling_page());
 490   guarantee(info != NULL, "Shouldn't be NULL");
 491   assert(os::is_poll_address(polling_page), "should be");
 492   __ mov(rscratch1, Address(polling_page, relocInfo::poll_type));
 493   add_debug_info_for_branch(info);  // This isn't just debug info:
 494   // it's the oop map
 495   __ read_polling_page(rscratch1, relocInfo::poll_type);
 496   return __ offset();
 497 }
 498 
 499 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) {
 500   if (from_reg != to_reg) {
 501     __ mov(to_reg, from_reg);
 502   }
 503 }
 504 
 505 void LIR_Assembler::swap_reg(Register a, Register b) {
 506   Unimplemented();
 507 }
 508 
 509 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
 510   assert(src->is_constant(), "should not call otherwise");
 511   assert(dest->is_register(), "should not call otherwise");
 512   LIR_Const* c = src->as_constant_ptr();
 513 
 514   switch (c->type()) {
 515     case T_INT: {
 516       assert(patch_code == lir_patch_none, "no patching handled here");
 517       __ mov(dest->as_register(), c->as_jint_bits());
 518       break;
 519     }
 520 
 521     case T_ADDRESS: {
 522       assert(patch_code == lir_patch_none, "no patching handled here");
 523       __ mov(dest->as_register(), c->as_jint());
 524       break;
 525     }
 526 
 527     case T_LONG: {
 528       assert(patch_code == lir_patch_none, "no patching handled here");
 529       __ mov(dest->as_register_lo(), c->as_jint_lo_bits());
 530       __ mov(dest->as_register_hi(), c->as_jint_hi_bits());
 531       break;
 532     }
 533 
 534     case T_OBJECT: {
 535         if (patch_code == lir_patch_none) {
 536           jobject2reg(c->as_jobject(), dest->as_register());
 537         } else {
 538           jobject2reg_with_patching(dest->as_register(), info);
 539         }
 540       break;
 541     }
 542 
 543     case T_METADATA: {
 544       if (patch_code != lir_patch_none) {
 545         klass2reg_with_patching(dest->as_register(), info);
 546       } else {
 547         __ mov_metadata(dest->as_register(), c->as_metadata());
 548       }
 549       break;
 550     }
 551 
 552     case T_FLOAT: {
 553         if(dest->is_single_fpu()) {
 554             if (__ operand_valid_for_float_immediate(c->as_jfloat())) {
 555                 __ vmov_f32(dest->as_float_reg(), c->as_jfloat());
 556             } else {
 557                 __ lea(rscratch1, InternalAddress(float_constant(c->as_jfloat())));
 558                 __ vldr_f32(dest->as_float_reg(), Address(rscratch1));
 559             }
 560         } else {
 561             assert(patch_code == lir_patch_none, "no patching handled here");
 562             __ mov(dest->as_register(), c->as_jint_bits());
 563         }
 564       break;
 565     }
 566 
 567     case T_DOUBLE: {
 568         if(dest->is_double_fpu()) {
 569             if (__ operand_valid_for_double_immediate(c->as_jdouble())) {
 570                 __ vmov_f64(dest->as_double_reg(), c->as_jdouble());
 571             } else {
 572                 __ lea(rscratch1, InternalAddress(double_constant(c->as_jdouble())));
 573                 __ vldr_f64(dest->as_double_reg(), Address(rscratch1));
 574             }
 575         } else {
 576             assert(patch_code == lir_patch_none, "no patching handled here");
 577             __ mov(dest->as_register_lo(), c->as_jint_lo_bits());
 578             __ mov(dest->as_register_hi(), c->as_jint_hi_bits());
 579         }
 580       break;
 581     }
 582 
 583     default:
 584       ShouldNotReachHere();
 585   }
 586 }
 587 
 588 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 589   LIR_Const* c = src->as_constant_ptr();
 590   switch (c->type()) {
 591   case T_OBJECT:
 592     {
 593       if (! c->as_jobject()) {
 594         __ mov(rscratch1, 0);
 595         __ str(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 596       } else {
 597         const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 598         reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 599       }
 600     }
 601     break;
 602   case T_ADDRESS:
 603     {
 604       const2reg(src, FrameMap::rscratch1_opr, lir_patch_none, NULL);
 605       reg2stack(FrameMap::rscratch1_opr, dest, c->type(), false);
 606     }
 607   case T_INT:
 608   case T_FLOAT:
 609     {
 610       __ mov(rscratch1, c->as_jint_bits());
 611       __ str(rscratch1, frame_map()->address_for_slot(dest->single_stack_ix()));
 612     }
 613     break;
 614   case T_LONG:
 615   case T_DOUBLE:
 616     {
 617         __ mov(rscratch1, c->as_jint_lo());
 618         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 619                                                         lo_word_offset_in_bytes));
 620         if (c->as_jint_lo() != c->as_jint_hi())
 621             __ mov(rscratch1, c->as_jint_hi());
 622         __ str(rscratch1, frame_map()->address_for_slot(dest->double_stack_ix(),
 623                                                         hi_word_offset_in_bytes));
 624     }
 625     break;
 626   default:
 627     ShouldNotReachHere();
 628   }
 629 }
 630 
 631 /*
 632  * For now this code can load only zero constants as in aarch32.
 633  * It seems like this implementation can break some tests in future.
 634  * TODO: ensure, write test, and rewrite if need.
 635  */
 636 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 637   assert(src->is_constant(), "should not call otherwise");
 638   LIR_Const* c = src->as_constant_ptr();
 639   LIR_Address* to_addr = dest->as_address_ptr();
 640 
 641   void (Assembler::* insn)(Register Rt, const Address &adr, Assembler::Condition cnd);
 642 
 643   __ mov(rscratch2, 0);
 644 
 645   int null_check_here = code_offset();
 646 
 647   Address::InsnDataType idt = Address::toInsnDataType(type);
 648   switch (type) {
 649   case T_ADDRESS:
 650     assert(c->as_jint() == 0, "should be");
 651     insn = &Assembler::str;
 652     break;
 653   case T_LONG: {
 654     assert(c->as_jlong() == 0, "should be");
 655     insn = &Assembler::str;
 656     Address addr = as_Address_hi(to_addr, Address::IDT_INT);
 657     null_check_here = code_offset();
 658     __ str(rscratch2, addr);
 659     idt = Address::IDT_INT;
 660     break;
 661   }
 662   case T_INT:
 663     assert(c->as_jint() == 0, "should be");
 664     insn = &Assembler::str;
 665     break;
 666   case T_OBJECT:
 667   case T_ARRAY:
 668     assert(c->as_jobject() == 0, "should be");
 669     insn = &Assembler::str;
 670     break;
 671   case T_CHAR:
 672   case T_SHORT:
 673     assert(c->as_jint() == 0, "should be");
 674     insn = &Assembler::strh;
 675     break;
 676   case T_BOOLEAN:
 677   case T_BYTE:
 678     assert(c->as_jint() == 0, "should be");
 679     insn = &Assembler::strb;
 680     break;
 681   default:
 682     ShouldNotReachHere();
 683   }
 684 
 685   (_masm->*insn)(rscratch2, as_Address(to_addr, idt), Assembler::C_DFLT);
 686   if (info) add_debug_info_for_null_check(null_check_here, info);
 687 }
 688 
 689 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) {
 690   assert(src->is_register(), "should not call otherwise");
 691   assert(dest->is_register(), "should not call otherwise");
 692 
 693   // move between cpu-registers
 694   if (dest->is_single_cpu()) {
 695     if (src->type() == T_LONG) {
 696       // Can do LONG -> OBJECT
 697       __ stop("investigate how \"LONG -> OBJECT\" works especially when high part is != 0");
 698       move_regs(src->as_register_lo(), dest->as_register());
 699       return;
 700     }
 701     if(src->is_single_fpu()) {
 702         __ vmov_f32(dest->as_register(), src->as_float_reg());
 703     } else {
 704         assert(src->is_single_cpu(), "must match");
 705         if (src->type() == T_OBJECT) {
 706           __ verify_oop(src->as_register());
 707         }
 708         move_regs(src->as_register(), dest->as_register());
 709     }
 710   } else if (dest->is_double_cpu()) {
 711       if(src->is_double_fpu()) {
 712         __ vmov_f64(dest->as_register_lo(), dest->as_register_hi(), src->as_double_reg());
 713       } else {
 714         assert(src->is_double_cpu(), "must match");
 715         Register f_lo = src->as_register_lo();
 716         Register f_hi = src->as_register_hi();
 717         Register t_lo = dest->as_register_lo();
 718         Register t_hi = dest->as_register_hi();
 719         assert(f_hi != f_lo, "must be different");
 720         assert(t_hi != t_lo, "must be different");
 721         check_register_collision(t_lo, &f_hi);
 722         move_regs(f_lo, t_lo);
 723         move_regs(f_hi, t_hi);
 724       }
 725   } else if (dest->is_single_fpu()) {
 726       if(src->is_single_cpu()) {
 727         __ vmov_f32(dest->as_float_reg(), src->as_register());
 728       } else {
 729         __ vmov_f32(dest->as_float_reg(), src->as_float_reg());
 730       }
 731   } else if (dest->is_double_fpu()) {
 732       if(src->is_double_cpu()) {
 733         __ vmov_f64(dest->as_double_reg(), src->as_register_lo(), src->as_register_hi());
 734       } else {
 735         __ vmov_f64(dest->as_double_reg(), src->as_double_reg());
 736       }
 737   } else {
 738     ShouldNotReachHere();
 739   }
 740 }
 741 
 742 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
 743   if (src->is_single_cpu()) {
 744     if (type == T_ARRAY || type == T_OBJECT) {
 745       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 746       __ verify_oop(src->as_register());
 747     } else {
 748       __ str(src->as_register(), frame_map()->address_for_slot(dest->single_stack_ix()));
 749     }
 750 
 751   } else if (src->is_double_cpu()) {
 752     Address dest_addr_LO = frame_map()->address_for_slot(dest->double_stack_ix(), lo_word_offset_in_bytes);
 753     __ strd(src->as_register_lo(), src->as_register_hi(), dest_addr_LO);
 754   } else if (src->is_single_fpu()) {
 755     Address dest_addr = frame_map()->address_for_slot(dest->single_stack_ix());
 756     __ vstr_f32(src->as_float_reg(), dest_addr.safe_for(Address::IDT_FLOAT, _masm, rscratch1));
 757   } else if (src->is_double_fpu()) {
 758     Address dest_addr = frame_map()->address_for_slot(dest->double_stack_ix());
 759     __ vstr_f64(src->as_double_reg(), dest_addr.safe_for(Address::IDT_DOUBLE, _masm, rscratch1));
 760   } else {
 761     ShouldNotReachHere();
 762   }
 763 
 764 }
 765 
 766 
 767 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide, bool /* unaligned */) {
 768   LIR_Address* to_addr = dest->as_address_ptr();
 769 
 770   if (type == T_ARRAY || type == T_OBJECT) {
 771     __ verify_oop(src->as_register());
 772   }
 773 
 774   PatchingStub* patch = NULL;
 775   if (patch_code != lir_patch_none) {
 776     assert(to_addr->disp() != 0, "must have");
 777 
 778     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 779     address const_addr = __ address_constant(0);
 780     if (!const_addr) BAILOUT("patchable offset");
 781     __ relocate(section_word_Relocation::spec(const_addr, CodeBuffer::SECT_CONSTS));
 782     __ patchable_load(rscratch1, const_addr);
 783     patching_epilog(patch, patch_code, to_addr->base()->as_register(), info);
 784 
 785     to_addr = new LIR_Address(to_addr->base(), FrameMap::rscratch1_opr, to_addr->type());
 786   }
 787 
 788 
 789   int null_check_here = code_offset();
 790   switch (type) {
 791     case T_FLOAT:
 792         if(src->is_single_fpu()) {
 793             Address addr = as_Address(to_addr, Address::IDT_FLOAT);
 794             null_check_here = code_offset();
 795             __ vstr_f32(src->as_float_reg(), addr);
 796             break;
 797         } // fall through at FPUless system
 798     case T_ARRAY:   // fall through
 799     case T_OBJECT:  // fall through
 800     case T_ADDRESS: // fall though
 801     case T_INT: {
 802       Address addr = as_Address(to_addr, Address::toInsnDataType(type));
 803       null_check_here = code_offset();
 804       __ str(src->as_register(), addr);
 805       break;
 806     }
 807     case T_METADATA:
 808       // We get here to store a method pointer to the stack to pass to
 809       // a dtrace runtime call. This can't work on 64 bit with
 810       // compressed klass ptrs: T_METADATA can be a compressed klass
 811       // ptr or a 64 bit method pointer.
 812       ShouldNotReachHere();
 813 //      __ str(src->as_register(), as_Address(to_addr));
 814       break;
 815 
 816     case T_DOUBLE:
 817         if(src->is_double_fpu()) {
 818             Address addr = as_Address(to_addr, Address::IDT_DOUBLE);
 819             null_check_here = code_offset();
 820             __ vstr_f64(src->as_double_reg(), addr);
 821             break;
 822         } // fall through at FPUless system
 823     case T_LONG: {
 824       Address addr = as_Address_lo(to_addr, Address::IDT_LONG);
 825       null_check_here = code_offset();
 826       null_check_here += __ strd(src->as_register_lo(), src->as_register_hi(), addr);
 827       break;
 828     }
 829 
 830     case T_BYTE:    // fall through
 831     case T_BOOLEAN: {
 832       Address addr = as_Address(to_addr, Address::toInsnDataType(type));
 833       null_check_here = code_offset();
 834       __ strb(src->as_register(), addr);
 835       break;
 836     }
 837     case T_CHAR:    // fall through
 838     case T_SHORT: {
 839       Address addr = as_Address(to_addr, Address::toInsnDataType(type));
 840       null_check_here = code_offset();
 841       __ strh(src->as_register(), addr);
 842       break;
 843     }
 844     default:
 845       ShouldNotReachHere();
 846   }
 847 
 848   if (info != NULL) {
 849     add_debug_info_for_null_check(null_check_here, info);
 850   }
 851 }
 852 
 853 
 854 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
 855   assert(src->is_stack(), "should not call otherwise");
 856   assert(dest->is_register(), "should not call otherwise");
 857 
 858   if (dest->is_single_cpu()) {
 859     if (type == T_ARRAY || type == T_OBJECT) {
 860       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 861       __ verify_oop(dest->as_register());
 862     } else {
 863       __ ldr(dest->as_register(), frame_map()->address_for_slot(src->single_stack_ix()));
 864     }
 865 
 866   } else if (dest->is_double_cpu()) {
 867     Address src_addr_LO = frame_map()->address_for_slot(src->double_stack_ix(), lo_word_offset_in_bytes);
 868     __ ldrd(dest->as_register_lo(), dest->as_register_hi(), src_addr_LO);
 869   } else if (dest->is_single_fpu()) {
 870     Address src_addr = frame_map()->address_for_slot(src->single_stack_ix());
 871     __ vldr_f32(dest->as_float_reg(), src_addr.safe_for(Address::IDT_FLOAT, _masm, rscratch1));
 872   } else if (dest->is_double_fpu()) {
 873     Address src_addr = frame_map()->address_for_slot(src->double_stack_ix());
 874     __ vldr_f64(dest->as_double_reg(), src_addr.safe_for(Address::IDT_DOUBLE, _masm, rscratch1));
 875   } else {
 876     ShouldNotReachHere();
 877   }
 878 }
 879 
 880 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) {
 881   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id);
 882   __ relocate(metadata_Relocation::spec(__ oop_recorder()->allocate_metadata_index(NULL)));
 883   __ patchable_load(reg, pc());
 884   patching_epilog(patch, lir_patch_normal, reg, info);
 885 }
 886 
 887 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
 888 
 889   LIR_Opr temp;
 890   if (type == T_LONG || type == T_DOUBLE)
 891     temp = FrameMap::rscratch_long_opr;
 892   else
 893     temp = FrameMap::rscratch1_opr;
 894 
 895   stack2reg(src, temp, src->type());
 896   reg2stack(temp, dest, dest->type(), false);
 897 }
 898 
 899 
 900 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool /* unaligned */) {
 901   LIR_Address* from_addr = src->as_address_ptr();
 902 
 903   if (from_addr->base()->type() == T_OBJECT) {
 904     __ verify_oop(from_addr->base()->as_pointer_register());
 905   }
 906 
 907   PatchingStub* patch = NULL;
 908   if (patch_code != lir_patch_none) {
 909     assert(from_addr->disp() != 0, "must have");
 910 
 911     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
 912     address const_addr = __ address_constant(0);
 913     if (!const_addr) BAILOUT("patchable offset");
 914     __ relocate(section_word_Relocation::spec(const_addr, CodeBuffer::SECT_CONSTS));
 915     __ patchable_load(rscratch1, const_addr);
 916     patching_epilog(patch, patch_code, from_addr->base()->as_register(), info);
 917 
 918     from_addr = new LIR_Address(from_addr->base(), FrameMap::rscratch1_opr, from_addr->type());
 919   }
 920 
 921   int null_check_here = code_offset();
 922 
 923   switch (type) {
 924     case T_FLOAT:
 925         if(dest->is_single_fpu()){
 926             Address addr = as_Address(from_addr, Address::IDT_FLOAT);
 927             null_check_here = code_offset();
 928             __ vldr_f32(dest->as_float_reg(), addr);
 929               break;
 930         }  // fall through at FPUless systems
 931     case T_ARRAY:   // fall through
 932     case T_OBJECT:  // fall through
 933     case T_ADDRESS: // fall through
 934     case T_INT: {
 935       Address addr = as_Address(from_addr, Address::toInsnDataType(type));
 936       null_check_here = code_offset();
 937       __ ldr(dest->as_register(), addr);
 938       break;
 939     }
 940     case T_METADATA:
 941       // We get here to store a method pointer to the stack to pass to
 942       // a dtrace runtime call. This can't work on 64 bit with
 943       // compressed klass ptrs: T_METADATA can be a compressed klass
 944       // ptr or a 64 bit method pointer.
 945       ShouldNotReachHere();
 946 //      __ ldr(dest->as_register(), as_Address(from_addr));
 947       break;
 948     case T_DOUBLE:
 949         if(dest->is_double_fpu()){
 950             Address addr = as_Address(from_addr, Address::IDT_DOUBLE);
 951             null_check_here = code_offset();
 952             __ vldr_f64(dest->as_double_reg(), addr);
 953               break;
 954         } // fall through at FPUless systems
 955     case T_LONG: {
 956       Address addr = as_Address_lo(from_addr, Address::IDT_LONG);
 957       null_check_here = code_offset();
 958       null_check_here += __ ldrd(dest->as_register_lo(), dest->as_register_hi(), addr);
 959       break;
 960     }
 961 
 962     case T_BYTE: {
 963       Address addr =  as_Address(from_addr, Address::IDT_BYTE);
 964       null_check_here = code_offset();
 965       __ ldrsb(dest->as_register(), addr);
 966       break;
 967     }
 968     case T_BOOLEAN: {
 969       Address addr = as_Address(from_addr, Address::IDT_BOOLEAN);
 970       null_check_here = code_offset();
 971       __ ldrb(dest->as_register(), addr);
 972       break;
 973     }
 974 
 975     case T_CHAR: {
 976       Address addr = as_Address(from_addr, Address::IDT_CHAR);
 977       null_check_here = code_offset();
 978       __ ldrh(dest->as_register(), addr);
 979       break;
 980     }
 981     case T_SHORT: {
 982       Address addr = as_Address(from_addr, Address::IDT_SHORT);
 983       null_check_here = code_offset();
 984       __ ldrsh(dest->as_register(), addr);
 985       break;
 986     }
 987 
 988     default:
 989       ShouldNotReachHere();
 990   }
 991 
 992   if (type == T_ARRAY || type == T_OBJECT) {
 993     __ verify_oop(dest->as_register());
 994   }
 995 
 996   if (info != NULL) {
 997     add_debug_info_for_null_check(null_check_here, info);
 998   }
 999 }
1000 
1001 void LIR_Assembler::prefetchr(LIR_Opr src) {
1002   Unimplemented();
1003 }
1004 
1005 void LIR_Assembler::prefetchw(LIR_Opr src) {
1006   Unimplemented();
1007 }
1008 
1009 int LIR_Assembler::array_element_size(BasicType type) const {
1010   int elem_size = type2aelembytes(type);
1011   return exact_log2(elem_size);
1012 }
1013 
1014 void LIR_Assembler::emit_op3(LIR_Op3* op) {
1015   Register Rdividend = op->in_opr1()->as_register();
1016   Register Rdivisor  = op->in_opr2()->as_register();
1017   Register Rscratch  = op->in_opr3()->as_register();
1018   Register Rresult   = op->result_opr()->as_register();
1019   int divisor = -1;
1020 
1021   /*
1022   TODO: For some reason, using the Rscratch that gets passed in is
1023   not possible because the register allocator does not see the tmp reg
1024   as used, and assignes it the same register as Rdividend. We use rscratch1
1025    instead.
1026 
1027   assert(Rdividend != Rscratch, "");
1028   assert(Rdivisor  != Rscratch, "");
1029   */
1030 
1031   if (Rdivisor == noreg && is_power_of_2(divisor)) {
1032     // convert division by a power of two into some shifts and logical operations
1033   }
1034 
1035   assert(op->code() == lir_irem || op->code() == lir_idiv, "should be irem or idiv");
1036   bool want_remainder = op->code() == lir_irem;
1037 
1038   __ divide(Rresult, Rdividend, Rdivisor, 32, want_remainder);
1039 }
1040 
1041 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
1042 #ifdef ASSERT
1043   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
1044   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
1045   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
1046 #endif
1047 
1048   if (op->cond() == lir_cond_always) {
1049     if (op->info() != NULL) add_debug_info_for_branch(op->info());
1050     __ b(*(op->label()));
1051   } else {
1052     Assembler::Condition acond;
1053     if (op->code() == lir_cond_float_branch) {
1054       bool is_unordered = (op->ublock() == op->block());
1055       // Assembler::EQ does not permit unordered branches, so we add
1056       // another branch here.  Likewise, Assembler::NE does not permit
1057       // ordered branches.
1058       if (is_unordered && op->cond() == lir_cond_equal
1059           || !is_unordered && op->cond() == lir_cond_notEqual)
1060         __ b(*(op->ublock()->label()), Assembler::VS);
1061       switch(op->cond()) {
1062       case lir_cond_equal:        acond = Assembler::EQ; break;
1063       case lir_cond_notEqual:     acond = Assembler::NE; break;
1064       case lir_cond_less:         acond = (is_unordered ? Assembler::LT : Assembler::LO); break;
1065       case lir_cond_lessEqual:    acond = (is_unordered ? Assembler::LE : Assembler::LS); break;
1066       case lir_cond_greaterEqual: acond = (is_unordered ? Assembler::HS : Assembler::GE); break;
1067       case lir_cond_greater:      acond = (is_unordered ? Assembler::HI : Assembler::GT); break;
1068       default:                    ShouldNotReachHere();
1069       }
1070     } else {
1071       switch (op->cond()) {
1072         case lir_cond_equal:        acond = Assembler::EQ; break;
1073         case lir_cond_notEqual:     acond = Assembler::NE; break;
1074         case lir_cond_less:         acond = Assembler::LT; break;
1075         case lir_cond_greaterEqual: acond = Assembler::GE; break;
1076         case lir_cond_lessEqual:    acond = Assembler::LE; break;
1077         case lir_cond_greater:      acond = Assembler::GT; break;
1078         case lir_cond_belowEqual:   acond = Assembler::LS; break;
1079         case lir_cond_aboveEqual:   acond = Assembler::HS; break;
1080         default:                         ShouldNotReachHere();
1081       }
1082       if (op->type() == T_LONG) {
1083         // a special trick here to be able to effectively compare jlongs
1084         // for the lessEqual and greater conditions the jlong operands are swapped
1085         // during comparison and hence should use mirror condition in conditional
1086         // instruction
1087         // see LIR_Assembler::comp_op and LIR_Assembler::cmove
1088         switch (op->cond()) {
1089           case lir_cond_lessEqual: acond = Assembler::GE;  break;
1090           case lir_cond_greater: acond = Assembler::LT;    break;
1091         }
1092       }
1093     }
1094     __ b(*(op->label()), acond);
1095   }
1096 }
1097 
1098 FloatRegister LIR_Assembler::as_float_reg(LIR_Opr doubleReg) {
1099     assert(doubleReg->is_double_fpu(), "must be f64");
1100     return as_FloatRegister(doubleReg->fpu_regnrLo());
1101 }
1102 
1103 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
1104   LIR_Opr src  = op->in_opr();
1105   LIR_Opr dest = op->result_opr();
1106 
1107   switch (op->bytecode()) {
1108     case Bytecodes::_i2f:
1109       {
1110         __ vmov_f32(dest->as_float_reg(), src->as_register());
1111         __ vcvt_f32_s32(dest->as_float_reg(), dest->as_float_reg());
1112         break;
1113       }
1114     case Bytecodes::_i2d:
1115       {
1116         __ vmov_f32(as_float_reg(dest), src->as_register());
1117         __ vcvt_f64_s32(dest->as_double_reg(), as_float_reg(dest));
1118         break;
1119       }
1120     case Bytecodes::_f2d:
1121       {
1122         __ vcvt_f64_f32(dest->as_double_reg(), src->as_float_reg());
1123         break;
1124       }
1125     case Bytecodes::_d2f:
1126       {
1127         __ vcvt_f32_f64(dest->as_float_reg(), src->as_double_reg());
1128         break;
1129       }
1130     case Bytecodes::_i2c:
1131       {
1132         __ uxth(dest->as_register(), src->as_register());
1133         break;
1134       }
1135     case Bytecodes::_i2l:
1136       {
1137         const Register dst_hi = dest->as_register_hi();
1138         const Register dst_lo = dest->as_register_lo();
1139         const Register src_lo = as_reg(src);
1140         __ mov(dst_lo, src_lo);
1141         __ asr(dst_hi, src_lo, 31);
1142         break;
1143       }
1144     case Bytecodes::_i2s:
1145       {
1146         __ sxth(dest->as_register(), src->as_register());
1147         break;
1148       }
1149     case Bytecodes::_i2b:
1150       {
1151         __ sxtb(dest->as_register(), src->as_register());
1152         break;
1153       }
1154     case Bytecodes::_l2i:
1155       {
1156         assert(dest->is_single_cpu(), "must be single register");
1157         __ mov(dest->as_register(), src->as_register_lo());
1158         break;
1159       }
1160     case Bytecodes::_f2i:
1161       {
1162         __ vcvt_s32_f32(src->as_float_reg(), src->as_float_reg());
1163         __ vmov_f32(dest->as_register(), src->as_float_reg());
1164         break;
1165       }
1166     case Bytecodes::_d2i:
1167       {
1168         __ vcvt_s32_f64(as_float_reg(src), src->as_double_reg());
1169         __ vmov_f32(dest->as_register(), as_float_reg(src));
1170         break;
1171       }
1172     default: ShouldNotReachHere();
1173   }
1174 }
1175 
1176 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
1177   if (op->init_check()) {
1178     __ ldrb(rscratch1, Address(op->klass()->as_register(),
1179                                InstanceKlass::init_state_offset()));
1180     __ cmp(rscratch1, InstanceKlass::fully_initialized);
1181     add_debug_info_for_null_check_here(op->stub()->info());
1182     __ b(*op->stub()->entry(), Assembler::NE);
1183   }
1184   __ allocate_object(op->obj()->as_register(),
1185                      op->tmp1()->as_register(),
1186                      op->tmp2()->as_register(),
1187                      op->header_size(),
1188                      op->object_size(),
1189                      op->klass()->as_register(),
1190                      *op->stub()->entry());
1191   __ bind(*op->stub()->continuation());
1192 }
1193 
1194 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1195   Register len =  as_reg(op->len());
1196 
1197   if (UseSlowPath ||
1198       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
1199       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
1200     __ b(*op->stub()->entry());
1201   } else {
1202     Register tmp1 = op->tmp1()->as_register();
1203     Register tmp2 = op->tmp2()->as_register();
1204     Register tmp3 = op->tmp3()->as_register();
1205     if (len == tmp1) {
1206       tmp1 = tmp3;
1207     } else if (len == tmp2) {
1208       tmp2 = tmp3;
1209     } else if (len == tmp3) {
1210       // everything is ok
1211     } else {
1212       __ mov(tmp3, len);
1213     }
1214     __ allocate_array(op->obj()->as_register(),
1215                       len,
1216                       tmp1,
1217                       tmp2,
1218                       arrayOopDesc::header_size(op->type()),
1219                       array_element_size(op->type()),
1220                       op->klass()->as_register(),
1221                       *op->stub()->entry());
1222   }
1223   __ bind(*op->stub()->continuation());
1224 }
1225 
1226 void LIR_Assembler::type_profile_helper(Register mdo,
1227                                         ciMethodData *md, ciProfileData *data,
1228                                         Register recv, Label* update_done) {
1229   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1230     Label next_test;
1231     // See if the receiver is receiver[n].
1232     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1233     __ ldr(rscratch1, Address(rscratch2));
1234     __ cmp(recv, rscratch1);
1235     __ b(next_test, Assembler::NE);
1236     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)));
1237     __ addptr(data_addr, DataLayout::counter_increment);
1238     __ b(*update_done);
1239     __ bind(next_test);
1240   }
1241 
1242   // Didn't find receiver; find next empty slot and fill it in
1243   for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) {
1244     Label next_test;
1245     __ lea(rscratch2,
1246            Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))));
1247     Address recv_addr(rscratch2);
1248     __ ldr(rscratch1, recv_addr);
1249     __ cbnz(rscratch1, next_test);
1250     __ str(recv, recv_addr);
1251     __ mov(rscratch1, DataLayout::counter_increment);
1252     __ lea(rscratch2, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))));
1253     __ str(rscratch1, Address(rscratch2));
1254     __ b(*update_done);
1255     __ bind(next_test);
1256   }
1257 }
1258 
1259 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
1260   // we always need a stub for the failure case.
1261   CodeStub* stub = op->stub();
1262   Register obj = op->object()->as_register();
1263   Register k_RInfo = op->tmp1()->as_register();
1264   Register klass_RInfo = op->tmp2()->as_register();
1265   Register dst = op->result_opr()->as_register();
1266   ciKlass* k = op->klass();
1267   Register Rtmp1 = noreg;
1268 
1269   // check if it needs to be profiled
1270   ciMethodData* md;
1271   ciProfileData* data;
1272 
1273   if (op->should_profile()) {
1274     ciMethod* method = op->profiled_method();
1275     assert(method != NULL, "Should have method");
1276     int bci = op->profiled_bci();
1277     md = method->method_data_or_null();
1278     assert(md != NULL, "Sanity");
1279     data = md->bci_to_data(bci);
1280     assert(data != NULL,                "need data for type check");
1281     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1282   }
1283   Label profile_cast_success, profile_cast_failure;
1284   Label *success_target = op->should_profile() ? &profile_cast_success : success;
1285   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
1286 
1287   if (obj == k_RInfo) {
1288     k_RInfo = dst;
1289   } else if (obj == klass_RInfo) {
1290     klass_RInfo = dst;
1291   }
1292   if (k->is_loaded()) {
1293     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1294   } else {
1295     Rtmp1 = op->tmp3()->as_register();
1296     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1297   }
1298 
1299   assert_different_registers(obj, k_RInfo, klass_RInfo);
1300 
1301     if (op->should_profile()) {
1302       Label not_null;
1303       __ cbnz(obj, not_null);
1304       // Object is null; update MDO and exit
1305       Register mdo  = klass_RInfo;
1306       __ mov_metadata(mdo, md->constant_encoding());
1307       Address data_addr
1308         = __ form_address(rscratch2, mdo,
1309                           md->byte_offset_of_slot(data, DataLayout::DataLayout::header_offset()),
1310                           LogBytesPerWord);
1311       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1312       __ ldr(rscratch1, data_addr);
1313       __ orr(rscratch1, rscratch1, header_bits);
1314       __ str(rscratch1, data_addr);
1315       __ b(*obj_is_null);
1316       __ bind(not_null);
1317     } else {
1318       __ cbz(obj, *obj_is_null);
1319     }
1320 
1321   if (!k->is_loaded()) {
1322     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1323   } else {
1324     __ mov_metadata(k_RInfo, k->constant_encoding());
1325   }
1326   __ verify_oop(obj);
1327 
1328   if (op->fast_check()) {
1329     // get object class
1330     // not a safepoint as obj null check happens earlier
1331     __ load_klass(rscratch1, obj);
1332     __ cmp( rscratch1, k_RInfo);
1333 
1334     __ b(*failure_target, Assembler::NE);
1335     // successful cast, fall through to profile or jump
1336   } else {
1337     // get object class
1338     // not a safepoint as obj null check happens earlier
1339     __ load_klass(klass_RInfo, obj);
1340     if (k->is_loaded()) {
1341       // See if we get an immediate positive hit
1342       __ ldr(rscratch1, Address(klass_RInfo, long(k->super_check_offset())));
1343       __ cmp(k_RInfo, rscratch1);
1344       if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) {
1345         __ b(*failure_target, Assembler::NE);
1346         // successful cast, fall through to profile or jump
1347       } else {
1348         // See if we get an immediate positive hit
1349         __ b(*success_target, Assembler::EQ);
1350         // check for self
1351         __ cmp(klass_RInfo, k_RInfo);
1352         __ b(*success_target, Assembler::EQ);
1353 
1354         __ push(klass_RInfo);
1355         __ push(k_RInfo);
1356         __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1357         __ ldr(klass_RInfo, Address(__ post(sp, 2 * wordSize)));
1358 
1359         // result is a boolean
1360         __ cbz(klass_RInfo, *failure_target);
1361         // successful cast, fall through to profile or jump
1362       }
1363     } else {
1364       // perform the fast part of the checking logic
1365       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1366       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1367       __ push(klass_RInfo);
1368       __ push(k_RInfo);
1369       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1370       __ ldr(k_RInfo, Address(__ post(sp, 2 * wordSize)));
1371 
1372       // result is a boolean
1373       __ cbz(k_RInfo, *failure_target);
1374       // successful cast, fall through to profile or jump
1375     }
1376   }
1377   if (op->should_profile()) {
1378     Register mdo  = klass_RInfo, recv = k_RInfo;
1379     __ bind(profile_cast_success);
1380     __ mov_metadata(mdo, md->constant_encoding());
1381     __ load_klass(recv, obj);
1382     Label update_done;
1383     type_profile_helper(mdo, md, data, recv, success);
1384     __ b(*success);
1385 
1386     __ bind(profile_cast_failure);
1387     __ mov_metadata(mdo, md->constant_encoding());
1388     Address counter_addr
1389       = __ form_address(rscratch2, mdo,
1390                         md->byte_offset_of_slot(data, CounterData::count_offset()),
1391                         LogBytesPerWord);
1392     __ ldr(rscratch1, counter_addr);
1393     __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1394     __ str(rscratch1, counter_addr);
1395     __ b(*failure);
1396   }
1397   __ b(*success);
1398 }
1399 
1400 
1401 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
1402   LIR_Code code = op->code();
1403   if (code == lir_store_check) {
1404     Register value = op->object()->as_register();
1405     Register array = op->array()->as_register();
1406     Register k_RInfo = op->tmp1()->as_register();
1407     Register klass_RInfo = op->tmp2()->as_register();
1408     Register Rtmp1 = op->tmp3()->as_register();
1409 
1410     CodeStub* stub = op->stub();
1411 
1412     // check if it needs to be profiled
1413     ciMethodData* md;
1414     ciProfileData* data;
1415 
1416     if (op->should_profile()) {
1417       ciMethod* method = op->profiled_method();
1418       assert(method != NULL, "Should have method");
1419       int bci = op->profiled_bci();
1420       md = method->method_data_or_null();
1421       assert(md != NULL, "Sanity");
1422       data = md->bci_to_data(bci);
1423       assert(data != NULL,                "need data for type check");
1424       assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1425     }
1426     Label profile_cast_success, profile_cast_failure, done;
1427     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
1428     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
1429 
1430     if (op->should_profile()) {
1431       Label not_null;
1432       __ cbnz(value, not_null);
1433       // Object is null; update MDO and exit
1434       Register mdo  = klass_RInfo;
1435       __ mov_metadata(mdo, md->constant_encoding());
1436       Address data_addr
1437         = __ form_address(rscratch2, mdo,
1438                           md->byte_offset_of_slot(data, DataLayout::header_offset()),
1439                           LogBytesPerInt);
1440       int header_bits = DataLayout::flag_mask_to_header_mask(BitData::null_seen_byte_constant());
1441       __ ldr(rscratch1, data_addr);
1442       __ orr(rscratch1, rscratch1, header_bits);
1443       __ str(rscratch1, data_addr);
1444       __ b(done);
1445       __ bind(not_null);
1446     } else {
1447       __ cbz(value, done);
1448     }
1449 
1450     add_debug_info_for_null_check_here(op->info_for_exception());
1451     __ load_klass(k_RInfo, array);
1452     __ load_klass(klass_RInfo, value);
1453 
1454     // get instance klass (it's already uncompressed)
1455     __ ldr(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset()));
1456     // perform the fast part of the checking logic
1457     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, NULL);
1458     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
1459     __ push(klass_RInfo);
1460     __ push(k_RInfo);
1461     __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
1462     __ ldr(k_RInfo, Address(__ post(sp, 2 * wordSize)));
1463     // result is a boolean
1464     __ cbz(k_RInfo, *failure_target);
1465     // fall through to the success case
1466 
1467     if (op->should_profile()) {
1468       Register mdo  = klass_RInfo, recv = k_RInfo;
1469       __ bind(profile_cast_success);
1470       __ mov_metadata(mdo, md->constant_encoding());
1471       __ load_klass(recv, value);
1472       type_profile_helper(mdo, md, data, recv, &done);
1473       __ b(done);
1474 
1475       __ bind(profile_cast_failure);
1476       __ mov_metadata(mdo, md->constant_encoding());
1477       Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1478       __ lea(rscratch2, counter_addr);
1479       __ ldr(rscratch1, Address(rscratch2));
1480       __ sub(rscratch1, rscratch1, DataLayout::counter_increment);
1481       __ str(rscratch1, Address(rscratch2));
1482       __ b(*stub->entry());
1483     }
1484 
1485     __ bind(done);
1486   } else if (code == lir_checkcast) {
1487     Register obj = op->object()->as_register();
1488     Register dst = op->result_opr()->as_register();
1489     Label success;
1490     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
1491     __ bind(success);
1492     if (dst != obj) {
1493       __ mov(dst, obj);
1494     }
1495   } else if (code == lir_instanceof) {
1496     Register obj = op->object()->as_register();
1497     Register dst = op->result_opr()->as_register();
1498     Label success, failure, done;
1499     emit_typecheck_helper(op, &success, &failure, &failure);
1500     __ bind(failure);
1501     __ mov(dst, 0);
1502     __ b(done);
1503     __ bind(success);
1504     __ mov(dst, 1);
1505     __ bind(done);
1506   } else {
1507     ShouldNotReachHere();
1508   }
1509 }
1510 
1511 // TODO: reuse masm cmpxchgw
1512 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval, Register result) {
1513   assert(newval != cmpval, "must be different");
1514   Label retry_load, nope;
1515   // flush and load exclusive from the memory location
1516   // and fail if it is not what we expect
1517   __ bind(retry_load);
1518   __ ldrex(result, addr);
1519   __ cmp(result, cmpval);
1520   __ mov(result, 1, Assembler::NE);
1521   __ b(nope, Assembler::NE);
1522   // if we store+flush with no intervening write rscratch1 wil be zero
1523   __ strex(result, newval, addr);
1524   // retry so we only ever return after a load fails to compare
1525   // ensures we don't return a stale value after a failed write.
1526   __ cbnz(result, retry_load);
1527   __ membar(__ AnyAny);
1528   __ bind(nope);
1529 }
1530 
1531 void LIR_Assembler::casl(Register addr, Register newval_lo, Register newval_hi, Register cmpval_lo,  Register cmpval_hi,  Register tmp_lo, Register tmp_hi, Register result) {
1532   assert(newval_lo->successor() == newval_hi, "must be contiguous");
1533   assert(tmp_lo->successor() == tmp_hi, "must be contiguous");
1534   assert(tmp_lo->encoding_nocheck() % 2 == 0,  "Must be an even register");
1535   assert_different_registers(newval_lo, newval_hi, cmpval_lo,  cmpval_hi,  tmp_lo, tmp_hi);
1536 
1537   Label retry_load, nope;
1538   // flush and load exclusive from the memory location
1539   // and fail if it is not what we expect
1540   __ bind(retry_load);
1541   __ mov(result, 1);
1542   __ ldrexd(tmp_lo, addr);
1543   __ cmp(tmp_lo, cmpval_lo);
1544   __ b(nope, Assembler::NE);
1545   __ cmp(tmp_hi, cmpval_hi);
1546   __ b(nope, Assembler::NE);
1547   // if we store+flush with no intervening write rscratch1 wil be zero
1548   __ strexd(result, newval_lo, addr);
1549   // retry so we only ever return after a load fails to compare
1550   // ensures we don't return a stale value after a failed write.
1551   __ cbnz(result, retry_load);
1552   __ membar(__ AnyAny);
1553   __ bind(nope);
1554 }
1555 
1556 
1557 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1558   Register addr = as_reg(op->addr());
1559   Register result = as_reg(op->result_opr());
1560   if (op->code() == lir_cas_obj || op->code() == lir_cas_int) {
1561     Register newval = as_reg(op->new_value());
1562     Register cmpval = as_reg(op->cmp_value());
1563     casw(addr, newval, cmpval, result);
1564   } else if (op->code() == lir_cas_long){
1565     Register newval_lo = op->new_value()->as_register_lo();
1566     Register newval_hi = op->new_value()->as_register_hi();
1567     Register cmpval_lo = op->cmp_value()->as_register_lo();
1568     Register cmpval_hi = op->cmp_value()->as_register_hi();
1569     Register tmp_lo = op->tmp1()->as_register_lo();
1570     Register tmp_hi = op->tmp1()->as_register_hi();
1571     casl(addr, newval_lo, newval_hi, cmpval_lo, cmpval_hi, tmp_lo, tmp_hi, result);
1572   } else {
1573       ShouldNotReachHere();
1574   }
1575 }
1576 
1577 static void patch_condition(address start_insn, address end_insn, Assembler::Condition cond) {
1578   for (uint32_t* insn_p = (uint32_t*) start_insn; (address) insn_p < end_insn; ++insn_p) {
1579     uint32_t insn = *insn_p;
1580     assert((insn >> 28) == Assembler::AL, "instructions in patch"
1581      " should allow conditional form and be in ALWAYS condition");
1582     *insn_p = (insn & 0x0fffffff) | (cond << 28);
1583   }
1584 }
1585 
1586 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1587 
1588   Assembler::Condition acond, ncond;
1589   switch (condition) {
1590   case lir_cond_equal:        acond = Assembler::EQ; ncond = Assembler::NE; break;
1591   case lir_cond_notEqual:     acond = Assembler::NE; ncond = Assembler::EQ; break;
1592   case lir_cond_less:         acond = Assembler::LT; ncond = Assembler::GE; break;
1593   case lir_cond_greaterEqual: acond = Assembler::GE; ncond = Assembler::LT; break;
1594   case lir_cond_lessEqual:    acond = Assembler::LE; ncond = Assembler::GT; break;
1595   case lir_cond_greater:      acond = Assembler::GT; ncond = Assembler::LE; break;
1596   case lir_cond_belowEqual:   Unimplemented(); break;
1597   case lir_cond_aboveEqual:   Unimplemented(); break;
1598   default:                    ShouldNotReachHere();
1599   }
1600   if (type == T_LONG) {
1601       // for the lessEqual and greater conditions the jlong operands are swapped
1602       // during comparison and hence should use mirror condition in conditional
1603       // instruction. see comp_op())
1604     switch (condition) {
1605     case lir_cond_lessEqual:    acond = Assembler::GE; ncond = Assembler::LT; break;
1606     case lir_cond_greater:      acond = Assembler::LT; ncond = Assembler::GE; break;
1607     }
1608   }
1609 
1610   address true_instrs = __ pc();
1611   if (opr1->is_cpu_register()) {
1612     reg2reg(opr1, result);
1613   } else if (opr1->is_stack()) {
1614     stack2reg(opr1, result, result->type());
1615   } else if (opr1->is_constant()) {
1616     const2reg(opr1, result, lir_patch_none, NULL);
1617   } else {
1618     ShouldNotReachHere();
1619   }
1620   patch_condition(true_instrs, __ pc(), acond);
1621 
1622   address false_instrs = __ pc();
1623   if (opr2->is_cpu_register()) {
1624     reg2reg(opr2, result);
1625   } else if (opr2->is_stack()) {
1626     stack2reg(opr2, result, result->type());
1627   } else if (opr2->is_constant()) {
1628     const2reg(opr2, result, lir_patch_none, NULL);
1629   } else {
1630     ShouldNotReachHere();
1631   }
1632   patch_condition(false_instrs, __ pc(), ncond);
1633 }
1634 
1635 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1636   assert(info == NULL, "should never be used, idiv/irem and ldiv/lrem not handled by this method");
1637 
1638   if (left->is_single_cpu()) {
1639     assert(left->type() != T_FLOAT, "expect integer type");
1640     assert(right->type() != T_FLOAT, "expect integer type");
1641     assert(dest->type() != T_FLOAT, "expect integer type");
1642 
1643     Register lreg = left->as_register();
1644     Register dreg = as_reg(dest);
1645 
1646     if (right->is_single_cpu()) {
1647       // cpu register - cpu register
1648 
1649       assert((left->type() == T_INT || left->type() == T_OBJECT)
1650              && right->type() == T_INT
1651              && dest->type() == T_INT,
1652              "should be");
1653       Register rreg = right->as_register();
1654       switch (code) {
1655       case lir_add: __ add (dest->as_register(), lreg, rreg); break;
1656       case lir_sub: __ sub (dest->as_register(), lreg, rreg); break;
1657       case lir_mul: __ mul (dest->as_register(), lreg, rreg); break;
1658       default:      ShouldNotReachHere();
1659       }
1660 
1661     } else if (right->is_double_cpu()) {
1662       ShouldNotReachHere(); // for obj+long op the generator casts long to int before invoking add
1663     } else if (right->is_constant()) {
1664       // cpu register - constant
1665       jint c = right->as_constant_ptr()->as_jint();
1666 
1667       assert(code == lir_add || code == lir_sub || code == lir_mul, "mismatched arithmetic op");
1668       if (dreg == lreg && ( code != lir_mul && c == 0 || code == lir_mul && c == 1 ) ) {
1669         COMMENT("effective nop elided");
1670         return;
1671       }
1672 
1673       if (code != lir_mul && Assembler::operand_valid_for_add_sub_immediate(c)) {
1674         switch (code) {
1675         case lir_add: __ add(dreg, lreg, c); break;
1676         case lir_sub: __ sub(dreg, lreg, c); break;
1677         default: ShouldNotReachHere();
1678         }
1679       } else {
1680         __ mov(rscratch1, c);
1681         switch (code) {
1682         case lir_add: __ add(dreg, lreg, rscratch1); break;
1683         case lir_sub: __ sub(dreg, lreg, rscratch1); break;
1684         case lir_mul: __ mul(dreg, lreg, rscratch1); break;
1685         default: ShouldNotReachHere();
1686         }
1687       }
1688     } else {
1689       ShouldNotReachHere();
1690     }
1691 
1692   } else if (left->is_double_cpu()) {
1693     assert(left->type() != T_DOUBLE, "expect integer type");
1694     assert(right->type() != T_DOUBLE, "expect integer type");
1695     assert(dest->type() != T_DOUBLE, "expect integer type");
1696 
1697     Register lreg_lo = left->as_register_lo();
1698     Register lreg_hi = left->as_register_hi();
1699 
1700     if (right->is_double_cpu()) {
1701       // cpu register - cpu register
1702       Register rreg_lo = right->as_register_lo();
1703       Register rreg_hi = right->as_register_hi();
1704       Register dreg_lo = dest->as_register_lo();
1705       Register dreg_hi = dest->as_register_hi();
1706       if (code == lir_add || code == lir_sub) {
1707         check_register_collision(dreg_lo, &lreg_hi, &rreg_hi);
1708       }
1709       switch (code) {
1710       case lir_add: __ adds (dreg_lo, lreg_lo, rreg_lo);
1711                     __ adc (dreg_hi, lreg_hi, rreg_hi); break;
1712       case lir_sub: __ subs (dreg_lo, lreg_lo, rreg_lo);
1713                     __ sbc (dreg_hi, lreg_hi, rreg_hi); break;
1714       case lir_mul: __ mult_long (dreg_lo, dreg_hi,
1715                         lreg_lo, lreg_hi, rreg_lo, rreg_hi); break;
1716       default:
1717         ShouldNotReachHere();
1718       }
1719 
1720     } else if (right->is_constant()) {
1721       const jint c_lo = right->as_constant_ptr()->as_jint_lo_bits();
1722       const jint c_hi = right->as_constant_ptr()->as_jint_hi_bits();
1723       const Register dreg_lo = dest->as_register_lo();
1724       const Register dreg_hi = dest->as_register_hi();
1725       assert(code == lir_add || code == lir_sub, "mismatched arithmetic op");
1726       if (c_lo == 0 && c_hi == 0 && dreg_lo == lreg_lo && dreg_hi == lreg_hi) {
1727         COMMENT("effective nop elided");
1728         return;
1729       }
1730       check_register_collision(dreg_lo, &lreg_hi, NULL, rscratch2);
1731       switch (code) {
1732         case lir_add:
1733           if (Assembler::operand_valid_for_add_sub_immediate(c_lo))
1734             __ adds(dreg_lo, lreg_lo, c_lo);
1735           else {
1736             __ mov(rscratch1, c_lo);
1737             __ adds(dreg_lo, lreg_lo, rscratch1);
1738           }
1739           if (Assembler::operand_valid_for_add_sub_immediate(c_hi))
1740             __ adc(dreg_hi, lreg_hi, c_hi);
1741           else {
1742             __ mov(rscratch1, c_hi);
1743             __ adc(dreg_lo, lreg_hi, rscratch1);
1744           }
1745           break;
1746         case lir_sub:
1747           if (Assembler::operand_valid_for_add_sub_immediate(c_lo))
1748             __ subs(dreg_lo, lreg_lo, c_lo);
1749           else {
1750             __ mov(rscratch1, c_lo);
1751             __ subs(dreg_lo, lreg_lo, rscratch1);
1752           }
1753           if (Assembler::operand_valid_for_add_sub_immediate(c_hi))
1754             __ sbc(dreg_hi, lreg_hi, c_hi);
1755           else {
1756             __ mov(rscratch1, c_hi);
1757             __ sbc(dreg_hi, lreg_hi, rscratch1);
1758           }
1759           break;
1760         default:
1761           ShouldNotReachHere();
1762       }
1763     } else {
1764       ShouldNotReachHere();
1765     }
1766   } else if (left->is_single_fpu()) {
1767     assert(right->is_single_fpu(), "right hand side of float arithmetics needs to be float register");
1768     switch (code) {
1769     case lir_add: __ vadd_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1770     case lir_sub: __ vsub_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1771     case lir_mul: __ vmul_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1772     case lir_div: __ vdiv_f32 (dest->as_float_reg(), left->as_float_reg(), right->as_float_reg()); break;
1773     default:
1774       ShouldNotReachHere();
1775     }
1776   } else if (left->is_double_fpu()) {
1777     if (right->is_double_fpu()) {
1778       // cpu register - cpu register
1779       switch (code) {
1780       case lir_add: __ vadd_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1781       case lir_sub: __ vsub_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1782       case lir_mul: __ vmul_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1783       case lir_div: __ vdiv_f64 (dest->as_double_reg(), left->as_double_reg(), right->as_double_reg()); break;
1784       default:
1785         ShouldNotReachHere();
1786       }
1787     } else {
1788       if (right->is_constant()) {
1789         ShouldNotReachHere();
1790       }
1791       ShouldNotReachHere();
1792     }
1793   } else if (left->is_single_stack() || left->is_address()) {
1794     assert(left == dest, "left and dest must be equal");
1795     ShouldNotReachHere();
1796   } else {
1797     ShouldNotReachHere();
1798   }
1799 }
1800 
1801 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) {
1802   switch(code) {
1803   case lir_abs : __ vabs_f64(dest->as_double_reg(), value->as_double_reg()); break;
1804   case lir_sqrt: __ vsqrt_f64(dest->as_double_reg(), value->as_double_reg()); break;
1805   default      : ShouldNotReachHere();
1806   }
1807 }
1808 
1809 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) {
1810 
1811   assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register");
1812   Register Rleft = left->is_single_cpu() ? left->as_register() :
1813                                            left->as_register_lo();
1814    if (dst->is_single_cpu()) {
1815      Register Rdst = dst->as_register();
1816      if (right->is_constant()) {
1817        switch (code) {
1818          case lir_logic_and: __ andr (Rdst, Rleft, right->as_jint()); break;
1819          case lir_logic_or:  __ orr (Rdst, Rleft, right->as_jint()); break;
1820          case lir_logic_xor: __ eor (Rdst, Rleft, right->as_jint()); break;
1821          default: ShouldNotReachHere(); break;
1822        }
1823      } else {
1824        Register Rright = right->is_single_cpu() ? right->as_register() :
1825                                                   right->as_register_lo();
1826        switch (code) {
1827          case lir_logic_and: __ andr (Rdst, Rleft, Rright); break;
1828          case lir_logic_or:  __ orr (Rdst, Rleft, Rright); break;
1829          case lir_logic_xor: __ eor (Rdst, Rleft, Rright); break;
1830          default: ShouldNotReachHere(); break;
1831        }
1832      }
1833    } else {
1834      assert(dst->is_double_cpu(), "mismatched logic op operand size");
1835      const Register Rdst_lo = dst->as_register_lo();
1836      const Register Rdst_hi = dst->as_register_hi();
1837      Register Rleft_hi = left->as_register_hi();
1838      if (right->is_constant()) {
1839        // LIR generator enforces jlong constants to be valid_immediate12
1840        // so we know they fit into 32-bit int
1841        switch (code) {
1842          case lir_logic_and: __ andr (Rdst_lo, Rleft, (int)right->as_jlong()); break;
1843          case lir_logic_or:  __ orr (Rdst_lo, Rleft, (int)right->as_jlong()); break;
1844          case lir_logic_xor: __ eor (Rdst_lo, Rleft, (int)right->as_jlong()); break;
1845          default: ShouldNotReachHere(); break;
1846        }
1847      } else {
1848        assert(right->is_double_cpu(), "mismatched logic op operand size");
1849        Register Rright_lo = right->as_register_lo();
1850        Register Rright_hi = right->as_register_hi();
1851        check_register_collision(Rdst_lo, &Rleft_hi, &Rright_hi);
1852        switch (code) {
1853          case lir_logic_and: __ andr (Rdst_lo, Rleft, Rright_lo);
1854                              __ andr (Rdst_hi, Rleft_hi, Rright_hi); break;
1855          case lir_logic_or:  __ orr (Rdst_lo, Rleft, Rright_lo);
1856                              __ orr (Rdst_hi, Rleft_hi, Rright_hi); break;
1857          case lir_logic_xor: __ eor (Rdst_lo, Rleft, Rright_lo);
1858                              __ eor (Rdst_hi, Rleft_hi, Rright_hi); break;
1859          default: ShouldNotReachHere(); break;
1860        }
1861      }
1862    }
1863 }
1864 
1865 
1866 
1867 void LIR_Assembler::arithmetic_idiv(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr temp, LIR_Opr result, CodeEmitInfo* info) { Unimplemented(); }
1868 
1869 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1870   if (opr1->is_single_cpu()) {
1871 
1872     assert(opr1->type() != T_FLOAT, "expect integer type");// softfp guard
1873     assert(opr2->type() != T_FLOAT, "expect integer type");
1874 
1875     Register reg1 = as_reg(opr1);
1876     if (opr2->is_single_cpu()) {
1877       // cpu register - cpu register
1878       Register reg2 = opr2->as_register();
1879       __ cmp(reg1, reg2);
1880     } else if (opr2->is_constant()) {
1881       LIR_Const* c = opr2->as_constant_ptr();
1882       if (c->type() == T_INT) {
1883         __ cmp(reg1, c->as_jint(), rscratch1, Assembler::C_DFLT);
1884       } else if (c->type() == T_OBJECT || c->type() == T_ARRAY) {
1885         jobject o = c->as_jobject();
1886         if (o == NULL) {
1887           __ cmp(reg1, (int32_t)NULL_WORD);
1888         } else {
1889           __ movoop(rscratch1, o);
1890           __ cmpptr(reg1, rscratch1);
1891         }
1892       } else {
1893         fatal(err_msg("unexpected type: %s", basictype_to_str(c->type())));
1894       }
1895     } else if (opr2->is_address()) {
1896       __ ldr(rscratch2, as_Address(opr2->as_address_ptr(), rscratch1, Address::IDT_INT));
1897       __ cmp(reg1, rscratch2);
1898     } else {
1899       ShouldNotReachHere();
1900     }
1901 
1902   } else if (opr1->is_double_cpu()) {
1903     assert(opr1->type() == T_LONG, "expect jlong type");
1904     assert(opr2->type() == T_LONG, "expect jlong type");
1905     Register xlo = opr1->as_register_lo();
1906     Register xhi = opr1->as_register_hi();
1907     if (opr2->is_double_cpu()) {
1908       // cpu register - cpu register
1909       Register ylo = opr2->as_register_lo();
1910       Register yhi = opr2->as_register_hi();
1911       switch (condition) {
1912         case lir_cond_equal:
1913         case lir_cond_notEqual:
1914         case lir_cond_belowEqual:
1915         case lir_cond_aboveEqual:
1916           // these need APSR.ZC. the ops below set them correctly (but not APSR.V)
1917           __ cmp(xhi, yhi);
1918           __ cmp(xlo, ylo, Assembler::EQ);
1919           break;
1920         case lir_cond_less:
1921         case lir_cond_greaterEqual:
1922           __ cmp(xlo, ylo);
1923           __ sbcs(rscratch1, xhi, yhi);
1924           break;
1925         case lir_cond_lessEqual:
1926         case lir_cond_greater:
1927           // here goes a trick: the below operations do not produce the valid
1928           // value for the APSR.Z flag and there is no easy way to set it. so
1929           // we exchange the order of arguments in the comparison and use the
1930           // opposite condition in the conditional statement that follows.
1931           // GE should be used instead of LE and LT in place of GT.
1932           // the comp_op() could only be followed by: emit_opBranch(), cmove() and
1933           // emit_assert(). these are patched to be aware of this trick
1934           __ cmp(ylo, xlo);
1935           __ sbcs(rscratch1, yhi, xhi);
1936           break;
1937       }
1938     } else if (opr2->is_constant()) {
1939       jlong y = opr2->as_jlong();
1940       assert(Assembler::operand_valid_for_add_sub_immediate(y), "immediate overflow");
1941       switch (condition) {
1942         case lir_cond_equal:
1943         case lir_cond_notEqual:
1944         case lir_cond_belowEqual:
1945         case lir_cond_aboveEqual:
1946           __ cmp(xhi, (int)(y >> 32));
1947           __ cmp(xlo, (int)y, Assembler::EQ);
1948           break;
1949         case lir_cond_less:
1950         case lir_cond_greaterEqual:
1951           __ cmp(xlo, (int)y);
1952           __ sbcs(rscratch1, xhi, (int)(y >> 32));
1953           break;
1954         case lir_cond_lessEqual:
1955         case lir_cond_greater:
1956           __ rsbs(rscratch1, xlo, (int)y);
1957           __ rscs(rscratch1, xhi, (int)(y >> 32));
1958           break;
1959       }
1960     } else {
1961       ShouldNotReachHere();
1962     }
1963   } else if (opr1->is_single_fpu()) {
1964     FloatRegister reg1 = opr1->as_float_reg();
1965     assert(opr2->is_single_fpu(), "expect single float register");
1966     FloatRegister reg2 = opr2->as_float_reg();
1967     __ vcmp_f32(reg1, reg2);
1968     __ get_fpsr();
1969   } else if (opr1->is_double_fpu()) {
1970     FloatRegister reg1 = opr1->as_double_reg();
1971     assert(opr2->is_double_fpu(), "expect double float register");
1972     FloatRegister reg2 = opr2->as_double_reg();
1973     __ vcmp_f64(reg1, reg2);
1974     __ get_fpsr();
1975   } else {
1976     ShouldNotReachHere();
1977   }
1978 }
1979 
1980 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1981   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1982     bool is_unordered_less = (code == lir_ucmp_fd2i);
1983     if (left->is_single_fpu()) {
1984       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1985     } else if (left->is_double_fpu()) {
1986       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1987     } else {
1988       ShouldNotReachHere();
1989     }
1990   } else if (code == lir_cmp_l2i) {
1991     __ mov(dst->as_register(), 1);
1992     __ subs(rscratch1, left->as_register_lo(), right->as_register_lo());
1993     __ sbc(rscratch2, left->as_register_hi(), right->as_register_hi());
1994     __ orrs(rscratch1, rscratch1, rscratch2);
1995     __ mov(dst->as_register(), -1, Assembler::MI);
1996     __ mov(dst->as_register(), 0, Assembler::EQ);
1997   } else {
1998     ShouldNotReachHere();
1999   }
2000 }
2001 
2002 
2003 void LIR_Assembler::align_call(LIR_Code code) {  }
2004 
2005 
2006 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2007   __ trampoline_call(Address(op->addr(), rtype));
2008   add_call_info(code_offset(), op->info());
2009 }
2010 
2011 
2012 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2013   __ ic_call(op->addr());
2014   add_call_info(code_offset(), op->info());
2015 }
2016 
2017 
2018 /* Currently, vtable-dispatch is only enabled for sparc platforms */
2019 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
2020   ShouldNotReachHere();
2021 }
2022 
2023 
2024 void LIR_Assembler::emit_static_call_stub() {
2025   address call_pc = __ pc();
2026   address stub = __ start_a_stub(call_stub_size);
2027   if (stub == NULL) {
2028     bailout("static call stub overflow");
2029     return;
2030   }
2031 
2032   int start = __ offset();
2033 
2034   __ relocate(static_stub_Relocation::spec(call_pc));
2035   __ mov_metadata(rmethod, (Metadata*)NULL);
2036   __ movptr(rscratch1, 0);
2037   __ b(rscratch1);
2038 
2039   assert(__ offset() - start <= call_stub_size, "stub too big");
2040   __ end_a_stub();
2041 }
2042 
2043 
2044 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2045   assert(exceptionOop->as_register() == r0, "must match");
2046   assert(exceptionPC->as_register() == r3, "must match");
2047 
2048   // exception object is not added to oop map by LinearScan
2049   // (LinearScan assumes that no oops are in fixed registers)
2050   info->add_register_oop(exceptionOop);
2051   Runtime1::StubID unwind_id;
2052 
2053   // get current pc information
2054   // pc is only needed if the method has an exception handler, the unwind code does not need it.
2055   int pc_for_athrow_offset = __ offset();
2056   __ add(exceptionPC->as_register(), r15_pc, -8);
2057   add_call_info(pc_for_athrow_offset, info); // for exception handler
2058 
2059   __ verify_not_null_oop(r0);
2060   // search an exception handler (r0: exception oop, r3: throwing pc)
2061   if (compilation()->has_fpu_code()) {
2062     unwind_id = Runtime1::handle_exception_id;
2063   } else {
2064     unwind_id = Runtime1::handle_exception_nofpu_id;
2065   }
2066   __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id)));
2067 
2068   // FIXME: enough room for two byte trap   ????
2069   __ nop();
2070 }
2071 
2072 
2073 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2074   assert(exceptionOop->as_register() == r0, "must match");
2075 
2076   __ b(_unwind_handler_entry);
2077 }
2078 
2079 
2080 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2081   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2082   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2083 
2084   switch (left->type()) {
2085     case T_INT:
2086     case T_ADDRESS:
2087     case T_OBJECT:
2088       __ andr(rscratch1, count->as_register(), 0x1f);
2089       switch (code) {
2090         case lir_shl: __ lsl(dreg, lreg, rscratch1); break;
2091         case lir_shr: __ asr(dreg, lreg, rscratch1); break;
2092         case lir_ushr: __ lsr(dreg, lreg, rscratch1); break;
2093         default:
2094           ShouldNotReachHere();
2095           break;
2096       }
2097       break;
2098     case T_LONG:
2099     {
2100       Register lreg_hi = left->as_register_hi();
2101       Register dreg_hi = dest->as_register_hi();
2102       const int word_bits = 8 * wordSize;
2103 
2104       if (code == lir_shl || code == lir_ushr) {
2105         check_register_collision(dreg, &lreg, &lreg_hi, rscratch1);
2106         check_register_collision(dreg_hi, &lreg, &lreg_hi, rscratch2);
2107       }
2108 
2109       switch (code) {
2110         case lir_shl:
2111           __ andr(dreg, count->as_register(), 0x3f);
2112           __ sub(dreg_hi, dreg, word_bits);
2113           __ lsl(lreg_hi, lreg_hi, dreg);
2114           __ orr(lreg_hi, lreg_hi, lreg, lsl(dreg_hi));
2115           __ rsb(dreg_hi, dreg, word_bits);
2116           __ orr(dreg_hi, lreg_hi, lreg, lsr(dreg_hi));
2117           __ lsl(dreg, lreg, dreg);
2118           break;
2119         case lir_shr: {
2120           __ mov(rscratch2, lreg_hi);
2121           __ andr(rscratch1, count->as_register(), 0x3f);
2122           __ lsr(dreg, lreg, rscratch1);
2123           __ rsb(dreg_hi, rscratch1, word_bits);
2124           __ orr(dreg, dreg, rscratch2, lsl(dreg_hi));
2125           __ asr(dreg_hi, rscratch2, rscratch1);
2126           __ subs(rscratch1, rscratch1, word_bits);
2127           __ mov(dreg, rscratch2, asr(rscratch1), Assembler::GT);
2128         }
2129           break;
2130         case lir_ushr:
2131           __ andr(dreg, count->as_register(), 0x3f);
2132           __ lsr(lreg, lreg, dreg);
2133           __ rsb(dreg_hi, dreg, word_bits);
2134           __ orr(lreg, lreg, lreg_hi, lsl(dreg_hi));
2135           __ lsr(dreg_hi, lreg_hi, dreg);
2136           __ sub(dreg, dreg, word_bits);
2137           __ orr(dreg, lreg, lreg_hi, lsr(dreg));
2138           break;
2139         default:
2140           ShouldNotReachHere();
2141           break;
2142       }
2143     }
2144       break;
2145     default:
2146       ShouldNotReachHere();
2147       break;
2148   }
2149 }
2150 
2151 
2152 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2153   Register dreg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo();
2154   Register lreg = left->is_single_cpu() ? left->as_register() : left->as_register_lo();
2155 
2156   if (!count) {
2157       reg2reg(left, dest);
2158       return;
2159    }
2160 
2161   switch (left->type()) {
2162     case T_INT:
2163     case T_ADDRESS:
2164     case T_OBJECT:
2165       switch (code) {
2166         case lir_shl: __ lsl(dreg, lreg, count); break;
2167         case lir_shr: __ asr(dreg, lreg, count); break;
2168         case lir_ushr: __ lsr(dreg, lreg, count); break;
2169         default:
2170           ShouldNotReachHere();
2171           break;
2172       }
2173       break;
2174     case T_LONG: {
2175       Register lreg_hi = left->as_register_hi();
2176       Register dreg_hi = dest->as_register_hi();
2177       const int word_bits = 8 * wordSize;
2178 
2179       switch (code) {
2180         case lir_shl:
2181           if (count >= word_bits) {
2182             __ lsl(dreg_hi, lreg, count - word_bits);
2183             __ mov(dreg, 0);
2184           } else {
2185             check_register_collision(dreg_hi, &lreg);
2186             __ lsl(dreg_hi, lreg_hi, count);
2187             __ orr(dreg_hi, dreg_hi, lreg, lsr(word_bits - count));
2188             __ lsl(dreg, lreg, count);
2189           }
2190           break;
2191         case lir_shr:
2192           if (count >= word_bits) {
2193             __ asr(dreg, lreg_hi, count - word_bits);
2194             __ asr(dreg_hi, lreg_hi, word_bits);
2195           } else {
2196             check_register_collision(dreg, &lreg_hi);
2197             __ lsr(dreg, lreg, count);
2198             __ orr(dreg, dreg, lreg_hi, lsl(word_bits - count));
2199             __ asr(dreg_hi, lreg_hi, count);
2200           }
2201           break;
2202         case lir_ushr:
2203           if (count >= word_bits) {
2204             __ lsr(dreg, lreg_hi, count - word_bits);
2205             __ mov(dreg_hi, 0);
2206           } else {
2207             check_register_collision(dreg, &lreg_hi);
2208             __ lsr(dreg, lreg, count);
2209             __ orr(dreg, dreg, lreg_hi, lsl(word_bits - count));
2210             __ lsr(dreg_hi, lreg_hi, count);
2211           }
2212           break;
2213         default:
2214           ShouldNotReachHere();
2215           break;
2216       }
2217     }
2218       break;
2219     default:
2220       ShouldNotReachHere();
2221       break;
2222   }
2223 }
2224 
2225 
2226 void LIR_Assembler::store_parameter(Register r, int offset_from_sp_in_words) {
2227   assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
2228   int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
2229   assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2230   __ str (r, Address(sp, offset_from_sp_in_bytes));
2231 }
2232 
2233 
2234 void LIR_Assembler::store_parameter(jint c,     int offset_from_sp_in_words) {
2235   assert(offset_from_sp_in_words >= 0, "invalid offset from sp");
2236   int offset_from_sp_in_bytes = offset_from_sp_in_words * BytesPerWord;
2237   assert(offset_from_sp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2238   __ mov (rscratch1, c);
2239   __ str (rscratch1, Address(sp, offset_from_sp_in_bytes));
2240 }
2241 
2242 // This code replaces a call to arraycopy; no exception may
2243 // be thrown in this code, they must be thrown in the System.arraycopy
2244 // activation frame; we could save some checks if this would not be the case
2245 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2246   ciArrayKlass* default_type = op->expected_type();
2247   Register src = op->src()->as_register();
2248   Register dst = op->dst()->as_register();
2249   Register src_pos = op->src_pos()->as_register();
2250   Register dst_pos = op->dst_pos()->as_register();
2251   Register length  = op->length()->as_register();
2252   Register tmp = op->tmp()->as_register();
2253   // due to limited number of registers available and in order to simplify
2254   // the code we fix the registers used by the arguments to this intrinsic.
2255   // see the comment in LIRGenerator::do_ArrayCopy
2256   assert(src == j_rarg0, "assumed by implementation");
2257   assert(src_pos == j_rarg1, "assumed by implementation");
2258   assert(dst == j_rarg2, "assumed by implementation");
2259   assert(dst_pos == j_rarg3, "assumed by implementation");
2260   assert(length == r4, "assumed by implementation");
2261   assert(tmp == r5, "assumed by implementation");
2262 
2263   CodeStub* stub = op->stub();
2264   int flags = op->flags();
2265   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2266   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2267 
2268   // if we don't know anything, just go through the generic arraycopy
2269   if (default_type == NULL // || basic_type == T_OBJECT
2270       ) {
2271     Label done;
2272     assert(src == r1 && src_pos == r2, "mismatch in calling convention");
2273 
2274     // Save the arguments in case the generic arraycopy fails and we
2275     // have to fall back to the JNI stub
2276     // length must be stored at [sp] because it's also used as an argument to C function
2277     __ str(length,  Address(sp, 0*BytesPerWord));
2278     __ str(dst,     Address(sp, 1*BytesPerWord));
2279     __ str(dst_pos, Address(sp, 2*BytesPerWord));
2280     __ str(src_pos, Address(sp, 3*BytesPerWord));
2281     __ str(src,     Address(sp, 4*BytesPerWord));
2282 
2283     address C_entry = CAST_FROM_FN_PTR(address, Runtime1::arraycopy);
2284     address copyfunc_addr = StubRoutines::generic_arraycopy();
2285 
2286     // The arguments are in java calling convention so we shift them
2287     // to C convention
2288     assert(c_rarg0 == j_rarg3, "assumed in the code below");
2289     __ mov(rscratch1, c_rarg0);
2290     assert_different_registers(c_rarg0, j_rarg1, j_rarg2);
2291     __ mov(c_rarg0, j_rarg0);
2292     assert_different_registers(c_rarg1, j_rarg2, j_rarg3);
2293     __ mov(c_rarg1, j_rarg1);
2294     assert_different_registers(c_rarg2, j_rarg3);
2295     __ mov(c_rarg2, j_rarg2);
2296     __ mov(c_rarg3, rscratch1);
2297     // the below C function follows C calling convention,
2298     // so should put 5th arg to stack but it's already there. see above
2299 
2300     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2301       __ mov(rscratch1, RuntimeAddress(C_entry));
2302       __ bl(rscratch1);
2303     } else {
2304 #ifndef PRODUCT
2305       if (PrintC1Statistics) {
2306         __ increment(ExternalAddress((address)&Runtime1::_generic_arraycopystub_cnt));
2307       }
2308 #endif
2309       __ far_call(RuntimeAddress(copyfunc_addr));
2310     }
2311 
2312     __ cbz(r0, *stub->continuation());
2313 
2314     // Reload values from the stack so they are where the stub
2315     // expects them.
2316     __ ldr(length,  Address(sp, 0*BytesPerWord));
2317     __ ldr(dst,     Address(sp, 1*BytesPerWord));
2318     __ ldr(dst_pos, Address(sp, 2*BytesPerWord));
2319     __ ldr(src_pos, Address(sp, 3*BytesPerWord));
2320     __ ldr(src,     Address(sp, 4*BytesPerWord));
2321 
2322     if (copyfunc_addr != NULL) {
2323       // r0 is -1^K where K == partial copied count
2324       __ inv(rscratch1, r0);
2325       // adjust length down and src/end pos up by partial copied count
2326       __ sub(length, length, rscratch1);
2327       __ add(src_pos, src_pos, rscratch1);
2328       __ add(dst_pos, dst_pos, rscratch1);
2329     }
2330     __ b(*stub->entry());
2331 
2332     __ bind(*stub->continuation());
2333     return;
2334   }
2335 
2336   assert(default_type != NULL && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2337 
2338   int elem_size = type2aelembytes(basic_type);
2339   int scale = exact_log2(elem_size);
2340 
2341   Address src_length_addr = Address(src, arrayOopDesc::length_offset_in_bytes());
2342   Address dst_length_addr = Address(dst, arrayOopDesc::length_offset_in_bytes());
2343   Address src_klass_addr = Address(src, oopDesc::klass_offset_in_bytes());
2344   Address dst_klass_addr = Address(dst, oopDesc::klass_offset_in_bytes());
2345 
2346   // test for NULL
2347   if (flags & LIR_OpArrayCopy::src_null_check) {
2348     __ cbz(src, *stub->entry());
2349   }
2350   if (flags & LIR_OpArrayCopy::dst_null_check) {
2351     __ cbz(dst, *stub->entry());
2352   }
2353 
2354   // check if negative
2355   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2356     __ cmp(src_pos, 0);
2357     __ b(*stub->entry(), Assembler::LT);
2358   }
2359   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2360     __ cmp(dst_pos, 0);
2361     __ b(*stub->entry(), Assembler::LT);
2362   }
2363 
2364   if (flags & LIR_OpArrayCopy::length_positive_check) {
2365     __ cmp(length, 0);
2366     __ b(*stub->entry(), Assembler::LT);
2367   }
2368 
2369   if (flags & LIR_OpArrayCopy::src_range_check) {
2370     __ add(tmp, src_pos, length);
2371     __ ldr(rscratch1, src_length_addr);
2372     __ cmp(tmp, rscratch1);
2373     __ b(*stub->entry(), Assembler::HI);
2374   }
2375   if (flags & LIR_OpArrayCopy::dst_range_check) {
2376     __ add(tmp, dst_pos, length);
2377     __ ldr(rscratch1, dst_length_addr);
2378     __ cmp(tmp, rscratch1);
2379     __ b(*stub->entry(), Assembler::HI);
2380   }
2381 
2382   // FIXME: The logic in LIRGenerator::arraycopy_helper clears
2383   // length_positive_check if the source of our length operand is an
2384   // arraylength.  However, that arraylength might be zero, and the
2385   // stub that we're about to call contains an assertion that count !=
2386   // 0 .  So we make this check purely in order not to trigger an
2387   // assertion failure.
2388   __ cbz(length, *stub->continuation());
2389 
2390   if (flags & LIR_OpArrayCopy::type_check) {
2391     // We don't know the array types are compatible
2392     if (basic_type != T_OBJECT) {
2393       // Simple test for basic type arrays
2394       __ ldr(tmp, src_klass_addr);
2395       __ ldr(rscratch1, dst_klass_addr);
2396       __ cmp(tmp, rscratch1);
2397       __ b(*stub->entry(), Assembler::NE);
2398     } else {
2399       // For object arrays, if src is a sub class of dst then we can
2400       // safely do the copy.
2401       Label cont, slow;
2402 
2403       __ push(RegSet::of(src, dst), sp);
2404 
2405       __ load_klass(src, src);
2406       __ load_klass(dst, dst);
2407 
2408       __ check_klass_subtype_fast_path(src, dst, tmp, &cont, &slow, NULL);
2409 
2410       __ push(src); // sub
2411       __ push(dst); // super
2412       __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id)));
2413       // result on TOS
2414       __ pop(src); // result
2415       __ pop(dst);
2416 
2417       __ cbnz(src, cont);
2418 
2419       __ bind(slow);
2420       __ pop(RegSet::of(src, dst), sp);
2421 
2422       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2423       if (copyfunc_addr != NULL) { // use stub if available
2424         // src is not a sub class of dst so we have to do a
2425         // per-element check.
2426 
2427         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2428         if ((flags & mask) != mask) {
2429           // Check that at least both of them object arrays.
2430           assert(flags & mask, "one of the two should be known to be an object array");
2431 
2432           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2433             __ load_klass(tmp, src);
2434           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2435             __ load_klass(tmp, dst);
2436           }
2437           int lh_offset = in_bytes(Klass::layout_helper_offset());
2438           Address klass_lh_addr(tmp, lh_offset);
2439           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2440           __ ldr(rscratch1, klass_lh_addr);
2441           __ mov(rscratch2, objArray_lh);
2442           __ eor(rscratch1, rscratch1, rscratch2);
2443           __ cbnz(rscratch1, *stub->entry());
2444         }
2445 
2446        // Spill because stubs can use any register they like and it's
2447        // easier to restore just those that we care about.
2448         __ str(dst,     Address(sp, 0*BytesPerWord));
2449         __ str(dst_pos, Address(sp, 1*BytesPerWord));
2450         __ str(length,  Address(sp, 2*BytesPerWord));
2451         __ str(src_pos, Address(sp, 3*BytesPerWord));
2452         __ str(src,     Address(sp, 4*BytesPerWord));
2453 
2454         assert(dst_pos == r0, "assumed in the code below");
2455         __ mov(rscratch1, dst_pos); // save dst_pos which is r0
2456         __ lea(c_rarg0, Address(src, src_pos, lsl(scale)));
2457         __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2458         assert_different_registers(c_rarg0, dst, length);
2459         __ lea(c_rarg1, Address(dst, rscratch1, lsl(scale)));
2460         __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2461         assert_different_registers(c_rarg1, dst, length);
2462 
2463         __ load_klass(c_rarg2, dst);
2464         __ ldr(c_rarg2, Address(c_rarg2, ObjArrayKlass::element_klass_offset()));
2465         __ ldr(c_rarg3, Address(c_rarg2, Klass::super_check_offset_offset()));
2466         __ far_call(RuntimeAddress(copyfunc_addr));
2467 
2468 #ifndef PRODUCT
2469         if (PrintC1Statistics) {
2470           Label failed;
2471           __ cbnz(r0, failed);
2472           __ increment(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_cnt));
2473           __ bind(failed);
2474         }
2475 #endif
2476 
2477         __ cbz(r0, *stub->continuation());
2478 
2479 #ifndef PRODUCT
2480         if (PrintC1Statistics) {
2481           __ increment(ExternalAddress((address)&Runtime1::_arraycopy_checkcast_attempt_cnt));
2482         }
2483 #endif
2484         assert_different_registers(dst, dst_pos, length, src_pos, src, rscratch1);
2485         __ mov(rscratch1, r0);
2486 
2487         // Restore previously spilled arguments
2488         __ ldr(dst,     Address(sp, 0*BytesPerWord));
2489         __ ldr(dst_pos, Address(sp, 1*BytesPerWord));
2490         __ ldr(length,  Address(sp, 2*BytesPerWord));
2491         __ ldr(src_pos, Address(sp, 3*BytesPerWord));
2492         __ ldr(src,     Address(sp, 4*BytesPerWord));
2493 
2494         // return value is -1^K where K is partial copied count
2495         __ mvn(rscratch1, rscratch1);
2496         // adjust length down and src/end pos up by partial copied count
2497         __ sub(length, length, rscratch1);
2498         __ add(src_pos, src_pos, rscratch1);
2499         __ add(dst_pos, dst_pos, rscratch1);
2500       }
2501 
2502       __ b(*stub->entry());
2503 
2504       __ bind(cont);
2505       __ pop(RegSet::of(src, dst), sp);
2506     }
2507   }
2508 
2509 #ifdef ASSERT
2510   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2511     // Sanity check the known type with the incoming class.  For the
2512     // primitive case the types must match exactly with src.klass and
2513     // dst.klass each exactly matching the default type.  For the
2514     // object array case, if no type check is needed then either the
2515     // dst type is exactly the expected type and the src type is a
2516     // subtype which we can't check or src is the same array as dst
2517     // but not necessarily exactly of type default_type.
2518     Label known_ok, halt;
2519     __ mov_metadata(tmp, default_type->constant_encoding());
2520 
2521     if (basic_type != T_OBJECT) {
2522 
2523       __ ldr(rscratch1, dst_klass_addr);
2524       __ cmp(tmp, rscratch1);
2525       __ b(halt, Assembler::NE);
2526       __ ldr(rscratch1, src_klass_addr);
2527       __ cmp(tmp, rscratch1);
2528       __ b(known_ok, Assembler::EQ);
2529     } else {
2530       __ ldr(rscratch1, dst_klass_addr);
2531       __ cmp(tmp, rscratch1);
2532       __ b(known_ok, Assembler::EQ);
2533       __ cmp(src, dst);
2534       __ b(known_ok, Assembler::EQ);
2535     }
2536     __ bind(halt);
2537     __ stop("incorrect type information in arraycopy");
2538     __ bind(known_ok);
2539   }
2540 #endif
2541   // skip array copy stub
2542   // aarch32 stub has not checks for zero-length (while x86 has)
2543   __ cbz(length, *stub->continuation());
2544 
2545   assert(dst_pos == r0, "assumed in the code below");
2546   __ mov(rscratch1, dst_pos); // save r0
2547   __ lea(c_rarg0, Address(src, src_pos, lsl(scale)));
2548   __ add(c_rarg0, c_rarg0, arrayOopDesc::base_offset_in_bytes(basic_type));
2549   assert_different_registers(c_rarg0, dst, rscratch1, length);
2550   __ lea(c_rarg1, Address(dst, rscratch1, lsl(scale)));
2551   __ add(c_rarg1, c_rarg1, arrayOopDesc::base_offset_in_bytes(basic_type));
2552   assert_different_registers(c_rarg1, dst, length);
2553   __ mov(c_rarg2, length);
2554 
2555   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2556   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2557   const char *name;
2558   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2559 
2560   CodeBlob *cb = CodeCache::find_blob(entry);
2561   if (cb) {
2562     __ far_call(RuntimeAddress(entry));
2563   } else {
2564     __ call_VM_leaf(entry, 3);
2565   }
2566 
2567   __ bind(*stub->continuation());
2568 }
2569 
2570 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2571   Register obj = op->obj_opr()->as_register();  // may not be an oop
2572   Register hdr = op->hdr_opr()->as_register();
2573   Register lock = op->lock_opr()->as_register();
2574   if (!UseFastLocking) {
2575     __ b(*op->stub()->entry());
2576   } else if (op->code() == lir_lock) {
2577     Register scratch = noreg;
2578     if (UseBiasedLocking) {
2579       scratch = op->scratch_opr()->as_register();
2580     }
2581     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2582     // add debug info for NullPointerException only if one is possible
2583     int null_check_offset = __ lock_object(hdr, obj, lock, scratch, *op->stub()->entry());
2584     if (op->info() != NULL) {
2585       add_debug_info_for_null_check(null_check_offset, op->info());
2586     }
2587     // done
2588   } else if (op->code() == lir_unlock) {
2589     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2590     __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2591   } else {
2592     Unimplemented();
2593   }
2594   __ bind(*op->stub()->continuation());
2595 }
2596 
2597 
2598 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2599   ciMethod* method = op->profiled_method();
2600   int bci          = op->profiled_bci();
2601   ciMethod* callee = op->profiled_callee();
2602 
2603   // Update counter for all call types
2604   ciMethodData* md = method->method_data_or_null();
2605   assert(md != NULL, "Sanity");
2606   ciProfileData* data = md->bci_to_data(bci);
2607   assert(data->is_CounterData(), "need CounterData for calls");
2608   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2609   Register mdo  = op->mdo()->as_register();
2610   __ mov_metadata(mdo, md->constant_encoding());
2611   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
2612   Bytecodes::Code bc = method->java_code_at_bci(bci);
2613   const bool callee_is_static = callee->is_loaded() && callee->is_static();
2614   // Perform additional virtual call profiling for invokevirtual and
2615   // invokeinterface bytecodes
2616   if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
2617       !callee_is_static &&  // required for optimized MH invokes
2618       C1ProfileVirtualCalls) {
2619     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2620     Register recv = op->recv()->as_register();
2621     assert_different_registers(mdo, recv);
2622     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2623     ciKlass* known_klass = op->known_holder();
2624     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2625       // We know the type that will be seen at this call site; we can
2626       // statically update the MethodData* rather than needing to do
2627       // dynamic tests on the receiver type
2628 
2629       // NOTE: we should probably put a lock around this search to
2630       // avoid collisions by concurrent compilations
2631       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2632       uint i;
2633       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2634         ciKlass* receiver = vc_data->receiver(i);
2635         if (known_klass->equals(receiver)) {
2636           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2637           __ addptr(data_addr, DataLayout::counter_increment);
2638           return;
2639         }
2640       }
2641 
2642       // Receiver type not found in profile data; select an empty slot
2643 
2644       // Note that this is less efficient than it should be because it
2645       // always does a write to the receiver part of the
2646       // VirtualCallData rather than just the first time
2647       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2648         ciKlass* receiver = vc_data->receiver(i);
2649         if (receiver == NULL) {
2650           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)));
2651           __ mov_metadata(rscratch1, known_klass->constant_encoding());
2652           __ lea(rscratch2, recv_addr);
2653           __ str(rscratch1, Address(rscratch2));
2654           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)));
2655           __ addptr(data_addr, DataLayout::counter_increment);
2656           return;
2657         }
2658       }
2659     } else {
2660       __ load_klass(recv, recv);
2661       Label update_done;
2662       type_profile_helper(mdo, md, data, recv, &update_done);
2663       // Receiver did not match any saved receiver and there is no empty row for it.
2664       // Increment total counter to indicate polymorphic case.
2665       __ addptr(counter_addr, DataLayout::counter_increment);
2666 
2667       __ bind(update_done);
2668     }
2669   } else {
2670     // Static call
2671     __ addptr(counter_addr, DataLayout::counter_increment);
2672   }
2673 }
2674 
2675 
2676 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
2677   Unimplemented();
2678 }
2679 
2680 
2681 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
2682   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
2683 }
2684 
2685 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2686   assert(op->crc()->is_single_cpu(),  "crc must be register");
2687   assert(op->val()->is_single_cpu(),  "byte value must be register");
2688   assert(op->result_opr()->is_single_cpu(), "result must be register");
2689   Register crc = op->crc()->as_register();
2690   Register val = op->val()->as_register();
2691   Register res = op->result_opr()->as_register();
2692 
2693   assert_different_registers(val, crc, res);
2694   __ lea(res, ExternalAddress(StubRoutines::crc_table_addr()));
2695 
2696   __ inv(crc, crc);
2697   __ update_byte_crc32(crc, val, res);
2698   __ inv(res, crc);
2699 }
2700 
2701 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2702   COMMENT("emit_profile_type {");
2703   Register obj = op->obj()->as_register();
2704   Register tmp = op->tmp()->as_pointer_register();
2705   Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), noreg, Address::IDT_INT);
2706   ciKlass* exact_klass = op->exact_klass();
2707   intptr_t current_klass = op->current_klass();
2708   bool not_null = op->not_null();
2709   bool no_conflict = op->no_conflict();
2710 
2711   Label update, next, none;
2712 
2713   bool do_null = !not_null;
2714   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2715   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2716 
2717   assert(do_null || do_update, "why are we here?");
2718   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2719   assert(mdo_addr.base() != rscratch1, "wrong register");
2720 
2721   __ verify_oop(obj);
2722 
2723   if (tmp != obj) {
2724     __ mov(tmp, obj);
2725   }
2726   if (do_null) {
2727     __ cbnz(tmp, update);
2728     if (!TypeEntries::was_null_seen(current_klass)) {
2729       __ ldr(rscratch2, mdo_addr);
2730       __ orr(rscratch2, rscratch2, TypeEntries::null_seen);
2731       __ str(rscratch2, mdo_addr);
2732     }
2733     if (do_update) {
2734 #ifndef ASSERT
2735       __ b(next);
2736     }
2737 #else
2738       __ b(next);
2739     }
2740   } else {
2741     __ cbnz(tmp, update);
2742     __ stop("unexpected null obj");
2743 #endif
2744   }
2745 
2746   __ bind(update);
2747 
2748   if (do_update) {
2749 #ifdef ASSERT
2750     if (exact_klass != NULL) {
2751       Label ok;
2752       __ load_klass(tmp, tmp);
2753       __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2754       __ eor(rscratch1, tmp, rscratch1);
2755       __ cbz(rscratch1, ok);
2756       __ stop("exact klass and actual klass differ");
2757       __ bind(ok);
2758     }
2759 #endif
2760     if (!no_conflict) {
2761       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2762         if (exact_klass != NULL) {
2763           __ mov_metadata(tmp, exact_klass->constant_encoding());
2764         } else {
2765           __ load_klass(tmp, tmp);
2766         }
2767 
2768         __ ldr(rscratch2, mdo_addr);
2769         __ eor(tmp, tmp, rscratch2);
2770         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2771         // klass seen before, nothing to do. The unknown bit may have been
2772         // set already but no need to check.
2773         __ cbz(rscratch1, next);
2774 
2775         __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2776         __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2777 
2778         if (TypeEntries::is_type_none(current_klass)) {
2779           __ cbz(rscratch2, none);
2780           __ cmp(rscratch2, TypeEntries::null_seen);
2781           __ b(none, Assembler::EQ);
2782           // There is a chance that the checks above (re-reading profiling
2783           // data from memory) fail if another thread has just set the
2784           // profiling to this obj's klass
2785           __ dmb(Assembler::ISH);
2786           __ ldr(rscratch2, mdo_addr);
2787           __ eor(tmp, tmp, rscratch2);
2788           __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2789           __ cbz(rscratch1, next);
2790         }
2791       } else {
2792         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2793                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2794 
2795         __ ldr(tmp, mdo_addr);
2796         __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2797         __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2798       }
2799 
2800       // different than before. Cannot keep accurate profile.
2801       __ ldr(rscratch2, mdo_addr);
2802       __ orr(rscratch2, rscratch2, TypeEntries::type_unknown);
2803       __ str(rscratch2, mdo_addr);
2804 
2805       if (TypeEntries::is_type_none(current_klass)) {
2806         __ b(next);
2807 
2808         __ bind(none);
2809         // first time here. Set profile type.
2810         __ str(tmp, mdo_addr);
2811       }
2812     } else {
2813       // There's a single possible klass at this profile point
2814       assert(exact_klass != NULL, "should be");
2815       if (TypeEntries::is_type_none(current_klass)) {
2816         __ mov_metadata(tmp, exact_klass->constant_encoding());
2817         __ ldr(rscratch2, mdo_addr);
2818         __ eor(tmp, tmp, rscratch2);
2819         __ andr(rscratch1, tmp, TypeEntries::type_klass_mask);
2820         __ cbz(rscratch1, next);
2821 #ifdef ASSERT
2822         {
2823           Label ok;
2824           __ ldr(rscratch1, mdo_addr);
2825           __ cbz(rscratch1, ok);
2826           __ cmp(rscratch1, TypeEntries::null_seen);
2827           __ b(ok, Assembler::EQ);
2828           // may have been set by another thread
2829           __ dmb(Assembler::ISH);
2830           __ mov_metadata(rscratch1, exact_klass->constant_encoding());
2831           __ ldr(rscratch2, mdo_addr);
2832           __ eor(rscratch2, rscratch1, rscratch2);
2833           __ andr(rscratch2, rscratch2, TypeEntries::type_mask);
2834           __ cbz(rscratch2, ok);
2835 
2836           __ stop("unexpected profiling mismatch");
2837           __ bind(ok);
2838         }
2839 #endif
2840         // first time here. Set profile type.
2841         __ ldr(tmp, mdo_addr);
2842       } else {
2843         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2844                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2845 
2846         __ ldr(tmp, mdo_addr);
2847         __ andr(rscratch1, tmp, TypeEntries::type_unknown);
2848         __ cbnz(rscratch1, next); // already unknown. Nothing to do anymore.
2849 
2850         __ orr(tmp, tmp, TypeEntries::type_unknown);
2851         __ str(tmp, mdo_addr);
2852         // FIXME: Write barrier needed here?
2853       }
2854     }
2855 
2856     __ bind(next);
2857   }
2858   COMMENT("} emit_profile_type");
2859 }
2860 
2861 
2862 void LIR_Assembler::align_backward_branch_target() {
2863 }
2864 
2865 
2866 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
2867   if (left->is_single_cpu()) {
2868     assert(left->type() != T_FLOAT, "expect integer type");
2869     assert(dest->type() != T_FLOAT, "expect integer type");
2870     assert(dest->is_single_cpu(), "expect single result reg");
2871     __ neg(dest->as_register(), left->as_register());
2872   } else if (left->is_double_cpu()) {
2873     assert(left->type() != T_DOUBLE, "expect integer type");
2874     assert(dest->type() != T_DOUBLE, "expect integer type");
2875     assert(dest->is_double_cpu(), "expect double result reg");
2876     const Register l_lo = left->as_register_lo();
2877     Register l_hi = left->as_register_hi();
2878     check_register_collision(dest->as_register_lo(), &l_hi);
2879     __ rsbs(dest->as_register_lo(), l_lo, 0);
2880     __ rsc(dest->as_register_hi(), l_hi, 0);
2881   } else if (left->is_single_fpu()) {
2882     assert(dest->is_single_fpu(), "expect single float result reg");
2883     __ vneg_f32(dest->as_float_reg(), left->as_float_reg());
2884   } else if (left->is_double_fpu()) {
2885     assert(left->is_double_fpu(), "expect double float operand reg");
2886     assert(dest->is_double_fpu(), "expect double float result reg");
2887     __ vneg_f64(dest->as_double_reg(), left->as_double_reg());
2888   } else {
2889       ShouldNotReachHere();
2890   }
2891 }
2892 
2893 
2894 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest) {
2895   __ lea(dest->as_register(), as_Address(addr->as_address_ptr(), noreg, Address::IDT_LEA));
2896 }
2897 
2898 
2899 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
2900   assert(!tmp->is_valid(), "don't need temporary");
2901   CodeBlob *cb = CodeCache::find_blob(dest);
2902   if (cb) {
2903     __ far_call(RuntimeAddress(dest));
2904   } else {
2905     __ lea(rscratch1, RuntimeAddress(dest));
2906     __ bl(rscratch1);
2907   }
2908   if (info != NULL) {
2909     add_call_info_here(info);
2910   }
2911   __ maybe_isb();
2912 }
2913 
2914 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
2915   if (type == T_LONG || type == T_DOUBLE) {
2916     const LIR_Opr long_val = FrameMap::long0_opr;
2917 
2918     int null_check_offset = -1;
2919 
2920     if (src->is_register() && dest->is_address()) {
2921       // long1 reserved as temp by LinearScan::pd_add_temps
2922       const LIR_Opr long_tmp = FrameMap::long1_opr;
2923       __ lea(rscratch1, as_Address_lo(dest->as_address_ptr(), Address::IDT_LEA));
2924 
2925 
2926       if (src->is_double_fpu()) {
2927         assert(type == T_DOUBLE, "invalid register allocation");
2928         // long0 reserved as temp by LinearScan::pd_add_temps
2929         __ vmov_f64(long_val->as_register_lo(), long_val->as_register_hi(), src->as_double_reg());
2930       } else {
2931         assert(type == T_LONG && src->is_same_register(long_val), "T_LONG src should be in long0 (by LIRGenerator)");
2932       }
2933 
2934       null_check_offset = __ offset();
2935       __ atomic_strd(long_val->as_register_lo(), long_val->as_register_hi(), rscratch1,
2936           long_tmp->as_register_lo(), long_tmp->as_register_hi());
2937 
2938     } else if (src->is_address() && dest->is_register()) {
2939       __ lea(rscratch1, as_Address_lo(src->as_address_ptr(), Address::IDT_LEA));
2940 
2941       null_check_offset = __ offset();
2942       __ atomic_ldrd(long_val->as_register_lo(), long_val->as_register_hi(), rscratch1);
2943 
2944       if (dest->is_double_fpu()) {
2945         __ vmov_f64(dest->as_double_reg(), long_val->as_register_lo(), long_val->as_register_hi());
2946       } else {
2947         assert(type != T_LONG || dest->is_same_register(long_val), "T_LONG dest should be in long0 (by LIRGenerator)");
2948       }
2949     } else {
2950       Unimplemented();
2951     }
2952 
2953     if (info != NULL) {
2954       add_debug_info_for_null_check(null_check_offset, info);
2955     }
2956 
2957   } else {
2958     move_op(src, dest, type, lir_patch_none, info,
2959             /*pop_fpu_stack*/false, /*unaligned*/false, /*wide*/false);
2960   }
2961 }
2962 
2963 #ifdef ASSERT
2964 // emit run-time assertion
2965 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
2966   assert(op->code() == lir_assert, "must be");
2967 
2968   if (op->in_opr1()->is_valid()) {
2969     assert(op->in_opr2()->is_valid(), "both operands must be valid");
2970     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
2971   } else {
2972     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
2973     assert(op->condition() == lir_cond_always, "no other conditions allowed");
2974   }
2975 
2976   Label ok;
2977   if (op->condition() != lir_cond_always) {
2978     Assembler::Condition acond = Assembler::AL;
2979     switch (op->condition()) {
2980       case lir_cond_equal:        acond = Assembler::EQ;  break;
2981       case lir_cond_notEqual:     acond = Assembler::NE;  break;
2982       case lir_cond_less:         acond = Assembler::LT;  break;
2983       case lir_cond_greaterEqual: acond = Assembler::GE;  break;
2984       case lir_cond_lessEqual:    acond = Assembler::LE;  break;
2985       case lir_cond_greater:      acond = Assembler::GT;  break;
2986       case lir_cond_belowEqual:   acond = Assembler::LS;  break;
2987       case lir_cond_aboveEqual:   acond = Assembler::HS;  break;
2988       default:                    ShouldNotReachHere();
2989     }
2990     if (op->in_opr1()->type() == T_LONG) {
2991       // a special trick here to be able to effectively compare jlongs
2992       // for the lessEqual and greater conditions the jlong operands are swapped
2993       // during comparison and hence should use mirror condition in conditional
2994       // instruction
2995       // see LIR_Assembler::comp_op and LIR_Assembler::cmove
2996       switch (op->condition()) {
2997         case lir_cond_lessEqual:    acond = Assembler::GE;  break;
2998         case lir_cond_greater:      acond = Assembler::LT;  break;
2999       }
3000     }
3001     __ b(ok, acond);
3002   }
3003   if (op->halt()) {
3004     const char* str = __ code_string(op->msg());
3005     __ stop(str);
3006   } else {
3007     breakpoint();
3008   }
3009   __ bind(ok);
3010 }
3011 #endif
3012 
3013 #ifndef PRODUCT
3014 #define COMMENT(x)   do { __ block_comment(x); } while (0)
3015 #else
3016 #define COMMENT(x)
3017 #endif
3018 
3019 void LIR_Assembler::membar() {
3020   COMMENT("membar");
3021   __ membar(MacroAssembler::AnyAny);
3022 }
3023 
3024 void LIR_Assembler::membar_acquire() {
3025   __ membar(Assembler::LoadLoad|Assembler::LoadStore);
3026 }
3027 
3028 void LIR_Assembler::membar_release() {
3029   __ membar(Assembler::LoadStore|Assembler::StoreStore);
3030 }
3031 
3032 void LIR_Assembler::membar_loadload() {
3033   __ membar(Assembler::LoadLoad);
3034 }
3035 
3036 void LIR_Assembler::membar_storestore() {
3037   __ membar(MacroAssembler::StoreStore);
3038 }
3039 
3040 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); }
3041 
3042 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); }
3043 
3044 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3045   __ mov(result_reg->as_register(), rthread);
3046 }
3047 
3048 
3049 void LIR_Assembler::peephole(LIR_List *lir) {
3050 #if 0
3051   if (tableswitch_count >= max_tableswitches)
3052     return;
3053 
3054   /*
3055     This finite-state automaton recognizes sequences of compare-and-
3056     branch instructions.  We will turn them into a tableswitch.  You
3057     could argue that C1 really shouldn't be doing this sort of
3058     optimization, but without it the code is really horrible.
3059   */
3060 
3061   enum { start_s, cmp1_s, beq_s, cmp_s } state;
3062   int first_key, last_key = -2147483648;
3063   int next_key = 0;
3064   int start_insn = -1;
3065   int last_insn = -1;
3066   Register reg = noreg;
3067   LIR_Opr reg_opr;
3068   state = start_s;
3069 
3070   LIR_OpList* inst = lir->instructions_list();
3071   for (int i = 0; i < inst->length(); i++) {
3072     LIR_Op* op = inst->at(i);
3073     switch (state) {
3074     case start_s:
3075       first_key = -1;
3076       start_insn = i;
3077       switch (op->code()) {
3078       case lir_cmp:
3079         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3080         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3081         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3082             && opr2->is_constant()
3083             && opr2->type() == T_INT) {
3084           reg_opr = opr1;
3085           reg = opr1->as_register();
3086           first_key = opr2->as_constant_ptr()->as_jint();
3087           next_key = first_key + 1;
3088           state = cmp_s;
3089           goto next_state;
3090         }
3091         break;
3092       }
3093       break;
3094     case cmp_s:
3095       switch (op->code()) {
3096       case lir_branch:
3097         if (op->as_OpBranch()->cond() == lir_cond_equal) {
3098           state = beq_s;
3099           last_insn = i;
3100           goto next_state;
3101         }
3102       }
3103       state = start_s;
3104       break;
3105     case beq_s:
3106       switch (op->code()) {
3107       case lir_cmp: {
3108         LIR_Opr opr1 = op->as_Op2()->in_opr1();
3109         LIR_Opr opr2 = op->as_Op2()->in_opr2();
3110         if (opr1->is_cpu_register() && opr1->is_single_cpu()
3111             && opr1->as_register() == reg
3112             && opr2->is_constant()
3113             && opr2->type() == T_INT
3114             && opr2->as_constant_ptr()->as_jint() == next_key) {
3115           last_key = next_key;
3116           next_key++;
3117           state = cmp_s;
3118           goto next_state;
3119         }
3120       }
3121       }
3122       last_key = next_key;
3123       state = start_s;
3124       break;
3125     default:
3126       assert(false, "impossible state");
3127     }
3128     if (state == start_s) {
3129       if (first_key < last_key - 5L && reg != noreg) {
3130         {
3131           // printf("found run register %d starting at insn %d low value %d high value %d\n",
3132           //        reg->encoding(),
3133           //        start_insn, first_key, last_key);
3134           //   for (int i = 0; i < inst->length(); i++) {
3135           //     inst->at(i)->print();
3136           //     tty->print("\n");
3137           //   }
3138           //   tty->print("\n");
3139         }
3140 
3141         struct tableswitch *sw = &switches[tableswitch_count];
3142         sw->_insn_index = start_insn, sw->_first_key = first_key,
3143           sw->_last_key = last_key, sw->_reg = reg;
3144         inst->insert_before(last_insn + 1, new LIR_OpLabel(&sw->_after));
3145         {
3146           // Insert the new table of branches
3147           int offset = last_insn;
3148           for (int n = first_key; n < last_key; n++) {
3149             inst->insert_before
3150               (last_insn + 1,
3151                new LIR_OpBranch(lir_cond_always, T_ILLEGAL,
3152                                 inst->at(offset)->as_OpBranch()->label()));
3153             offset -= 2, i++;
3154           }
3155         }
3156         // Delete all the old compare-and-branch instructions
3157         for (int n = first_key; n < last_key; n++) {
3158           inst->remove_at(start_insn);
3159           inst->remove_at(start_insn);
3160         }
3161         // Insert the tableswitch instruction
3162         inst->insert_before(start_insn,
3163                             new LIR_Op2(lir_cmp, lir_cond_always,
3164                                         LIR_OprFact::intConst(tableswitch_count),
3165                                         reg_opr));
3166         inst->insert_before(start_insn + 1, new LIR_OpLabel(&sw->_branches));
3167         tableswitch_count++;
3168       }
3169       reg = noreg;
3170       last_key = -2147483648;
3171     }
3172   next_state:
3173     ;
3174   }
3175 #endif
3176 }
3177 
3178 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) {
3179   BasicType type = src->type();
3180   Address addr = as_Address(src->as_address_ptr(), Address::toInsnDataType(type));
3181 
3182   bool is_long = false;
3183 
3184   switch(type) {
3185   case T_INT:
3186   case T_OBJECT:
3187   case T_ARRAY:
3188     break;
3189   case T_LONG:
3190     is_long = true;
3191     break;
3192   default:
3193     ShouldNotReachHere();
3194   }
3195 
3196   switch (code) {
3197   case lir_xadd:
3198     {
3199       Register tmp = tmp_op->as_register();
3200       Register dst = as_reg(dest);
3201       Label again;
3202       __ lea(tmp, addr);
3203       __ bind(again);
3204       if(is_long) {
3205           assert(dest->as_register_lo()->successor() == dest->as_register_hi(), "must be contiguous");
3206           assert((dest->as_register_lo()->encoding() & 1) == 0, "must be even");
3207           _masm->ldrexd(dst, tmp);
3208       } else {
3209           _masm->ldrex(dst, tmp);
3210       }
3211       arith_op(lir_add, dest, data, dest, NULL, false);
3212       if (is_long) {
3213         _masm->strexd(rscratch1, dst, tmp);
3214       } else {
3215         _masm->strex(rscratch1, dst, tmp);
3216       }
3217       __ cbnz(rscratch1, again);
3218       arith_op(lir_sub, dest, data, dest, NULL, false);
3219       break;
3220     }
3221   case lir_xchg:
3222     {
3223       Register tmp = tmp_op->as_register();
3224       Register obj = as_reg(data);
3225       Register dst = as_reg(dest);
3226       assert_different_registers(obj, addr.base(), tmp, rscratch1, dst);
3227       Label again;
3228       __ lea(tmp, addr);
3229       __ bind(again);
3230       if(is_long) {
3231           assert(dest->as_register_lo()->successor() == dest->as_register_hi(), "must be contiguous");
3232           assert((dest->as_register_lo()->encoding() & 1) == 0, "must be even");
3233 
3234           assert(data->is_double_cpu(), "should be double register");
3235           assert(data->as_register_lo()->successor() == data->as_register_hi(), "must be contiguous");
3236           assert((data->as_register_lo()->encoding() & 1) == 0, "must be even");
3237 
3238           _masm->ldrexd(dst, tmp);
3239           _masm->strexd(rscratch1, obj, tmp);
3240       } else {
3241          _masm->ldrex(dst, tmp);
3242          _masm->strex(rscratch1, obj, tmp);
3243       }
3244       __ cbnz(rscratch1, again);
3245     }
3246     break;
3247   default:
3248     ShouldNotReachHere();
3249   }
3250   __ membar(__ AnyAny);
3251 }
3252 
3253 void LIR_Assembler::check_register_collision(Register d, Register *s1, Register *s2, Register tmp) {
3254   // use a temp if any of the registers used as a source of operation
3255   // collide with result register of the prerequisite operation
3256   if (d == *s1) {
3257     __ mov(tmp, d);
3258     *s1 = tmp;
3259   } else if (s2 && d == *s2) {
3260     __ mov(tmp, d);
3261     *s2 = tmp;
3262   }
3263 }
3264 
3265 #undef __