1 /*
   2  * Copyright (c) 2000, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_Compilation.hpp"
  28 #include "c1/c1_LIRAssembler.hpp"
  29 #include "c1/c1_MacroAssembler.hpp"
  30 #include "c1/c1_Runtime1.hpp"
  31 #include "c1/c1_ValueStack.hpp"
  32 #include "ci/ciArrayKlass.hpp"
  33 #include "ci/ciInstance.hpp"
  34 #include "gc/shared/barrierSet.hpp"
  35 #include "gc/shared/cardTableBarrierSet.hpp"
  36 #include "gc/shared/collectedHeap.hpp"
  37 #include "memory/universe.hpp"
  38 #include "nativeInst_sparc.hpp"
  39 #include "oops/objArrayKlass.hpp"
  40 #include "runtime/frame.inline.hpp"
  41 #include "runtime/interfaceSupport.inline.hpp"
  42 #include "runtime/jniHandles.inline.hpp"
  43 #include "runtime/safepointMechanism.inline.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 
  46 #define __ _masm->
  47 
  48 
  49 //------------------------------------------------------------
  50 
  51 
  52 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
  53   if (opr->is_constant()) {
  54     LIR_Const* constant = opr->as_constant_ptr();
  55     switch (constant->type()) {
  56       case T_INT: {
  57         jint value = constant->as_jint();
  58         return Assembler::is_simm13(value);
  59       }
  60 
  61       default:
  62         return false;
  63     }
  64   }
  65   return false;
  66 }
  67 
  68 
  69 bool LIR_Assembler::is_single_instruction(LIR_Op* op) {
  70   switch (op->code()) {
  71     case lir_null_check:
  72     return true;
  73 
  74 
  75     case lir_add:
  76     case lir_ushr:
  77     case lir_shr:
  78     case lir_shl:
  79       // integer shifts and adds are always one instruction
  80       return op->result_opr()->is_single_cpu();
  81 
  82 
  83     case lir_move: {
  84       LIR_Op1* op1 = op->as_Op1();
  85       LIR_Opr src = op1->in_opr();
  86       LIR_Opr dst = op1->result_opr();
  87 
  88       if (src == dst) {
  89         NEEDS_CLEANUP;
  90         // this works around a problem where moves with the same src and dst
  91         // end up in the delay slot and then the assembler swallows the mov
  92         // since it has no effect and then it complains because the delay slot
  93         // is empty.  returning false stops the optimizer from putting this in
  94         // the delay slot
  95         return false;
  96       }
  97 
  98       // don't put moves involving oops into the delay slot since the VerifyOops code
  99       // will make it much larger than a single instruction.
 100       if (VerifyOops) {
 101         return false;
 102       }
 103 
 104       if (src->is_double_cpu() || dst->is_double_cpu() || op1->patch_code() != lir_patch_none ||
 105           ((src->is_double_fpu() || dst->is_double_fpu()) && op1->move_kind() != lir_move_normal)) {
 106         return false;
 107       }
 108 
 109       if (UseCompressedOops) {
 110         if (dst->is_address() && !dst->is_stack() && (dst->type() == T_OBJECT || dst->type() == T_ARRAY)) return false;
 111         if (src->is_address() && !src->is_stack() && (src->type() == T_OBJECT || src->type() == T_ARRAY)) return false;
 112       }
 113 
 114       if (UseCompressedClassPointers) {
 115         if (src->is_address() && !src->is_stack() && src->type() == T_ADDRESS &&
 116             src->as_address_ptr()->disp() == oopDesc::klass_offset_in_bytes()) return false;
 117       }
 118 
 119       if (dst->is_register()) {
 120         if (src->is_address() && Assembler::is_simm13(src->as_address_ptr()->disp())) {
 121           return !PatchALot;
 122         } else if (src->is_single_stack()) {
 123           return true;
 124         }
 125       }
 126 
 127       if (src->is_register()) {
 128         if (dst->is_address() && Assembler::is_simm13(dst->as_address_ptr()->disp())) {
 129           return !PatchALot;
 130         } else if (dst->is_single_stack()) {
 131           return true;
 132         }
 133       }
 134 
 135       if (dst->is_register() &&
 136           ((src->is_register() && src->is_single_word() && src->is_same_type(dst)) ||
 137            (src->is_constant() && LIR_Assembler::is_small_constant(op->as_Op1()->in_opr())))) {
 138         return true;
 139       }
 140 
 141       return false;
 142     }
 143 
 144     default:
 145       return false;
 146   }
 147   ShouldNotReachHere();
 148 }
 149 
 150 
 151 LIR_Opr LIR_Assembler::receiverOpr() {
 152   return FrameMap::O0_oop_opr;
 153 }
 154 
 155 
 156 LIR_Opr LIR_Assembler::osrBufferPointer() {
 157   return FrameMap::I0_opr;
 158 }
 159 
 160 
 161 int LIR_Assembler::initial_frame_size_in_bytes() const {
 162   return in_bytes(frame_map()->framesize_in_bytes());
 163 }
 164 
 165 
 166 // inline cache check: the inline cached class is in G5_inline_cache_reg(G5);
 167 // we fetch the class of the receiver (O0) and compare it with the cached class.
 168 // If they do not match we jump to slow case.
 169 int LIR_Assembler::check_icache() {
 170   int offset = __ offset();
 171   __ inline_cache_check(O0, G5_inline_cache_reg);
 172   return offset;
 173 }
 174 
 175 void LIR_Assembler::clinit_barrier(ciMethod* method) {
 176   ShouldNotReachHere(); // not implemented
 177 }
 178 
 179 void LIR_Assembler::osr_entry() {
 180   // On-stack-replacement entry sequence (interpreter frame layout described in interpreter_sparc.cpp):
 181   //
 182   //   1. Create a new compiled activation.
 183   //   2. Initialize local variables in the compiled activation.  The expression stack must be empty
 184   //      at the osr_bci; it is not initialized.
 185   //   3. Jump to the continuation address in compiled code to resume execution.
 186 
 187   // OSR entry point
 188   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 189   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 190   ValueStack* entry_state = osr_entry->end()->state();
 191   int number_of_locks = entry_state->locks_size();
 192 
 193   // Create a frame for the compiled activation.
 194   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 195 
 196   // OSR buffer is
 197   //
 198   // locals[nlocals-1..0]
 199   // monitors[number_of_locks-1..0]
 200   //
 201   // locals is a direct copy of the interpreter frame so in the osr buffer
 202   // so first slot in the local array is the last local from the interpreter
 203   // and last slot is local[0] (receiver) from the interpreter
 204   //
 205   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 206   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 207   // in the interpreter frame (the method lock if a sync method)
 208 
 209   // Initialize monitors in the compiled activation.
 210   //   I0: pointer to osr buffer
 211   //
 212   // All other registers are dead at this point and the locals will be
 213   // copied into place by code emitted in the IR.
 214 
 215   Register OSR_buf = osrBufferPointer()->as_register();
 216   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 217     int monitor_offset = BytesPerWord * method()->max_locals() +
 218       (2 * BytesPerWord) * (number_of_locks - 1);
 219     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 220     // the OSR buffer using 2 word entries: first the lock and then
 221     // the oop.
 222     for (int i = 0; i < number_of_locks; i++) {
 223       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 224 #ifdef ASSERT
 225       // verify the interpreter's monitor has a non-null object
 226       {
 227         Label L;
 228         __ ld_ptr(OSR_buf, slot_offset + 1*BytesPerWord, O7);
 229         __ cmp_and_br_short(O7, G0, Assembler::notEqual, Assembler::pt, L);
 230         __ stop("locked object is NULL");
 231         __ bind(L);
 232       }
 233 #endif // ASSERT
 234       // Copy the lock field into the compiled activation.
 235       __ ld_ptr(OSR_buf, slot_offset + 0, O7);
 236       __ st_ptr(O7, frame_map()->address_for_monitor_lock(i));
 237       __ ld_ptr(OSR_buf, slot_offset + 1*BytesPerWord, O7);
 238       __ st_ptr(O7, frame_map()->address_for_monitor_object(i));
 239     }
 240   }
 241 }
 242 
 243 
 244 // --------------------------------------------------------------------------------------------
 245 
 246 void LIR_Assembler::monitorexit(LIR_Opr obj_opr, LIR_Opr lock_opr, Register hdr, int monitor_no) {
 247   if (!GenerateSynchronizationCode) return;
 248 
 249   Register obj_reg = obj_opr->as_register();
 250   Register lock_reg = lock_opr->as_register();
 251 
 252   Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no);
 253   Register reg = mon_addr.base();
 254   int offset = mon_addr.disp();
 255   // compute pointer to BasicLock
 256   if (mon_addr.is_simm13()) {
 257     __ add(reg, offset, lock_reg);
 258   }
 259   else {
 260     __ set(offset, lock_reg);
 261     __ add(reg, lock_reg, lock_reg);
 262   }
 263   // unlock object
 264   MonitorAccessStub* slow_case = new MonitorExitStub(lock_opr, UseFastLocking, monitor_no);
 265   // _slow_case_stubs->append(slow_case);
 266   // temporary fix: must be created after exceptionhandler, therefore as call stub
 267   _slow_case_stubs->append(slow_case);
 268   if (UseFastLocking) {
 269     // try inlined fast unlocking first, revert to slow locking if it fails
 270     // note: lock_reg points to the displaced header since the displaced header offset is 0!
 271     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
 272     __ unlock_object(hdr, obj_reg, lock_reg, *slow_case->entry());
 273   } else {
 274     // always do slow unlocking
 275     // note: the slow unlocking code could be inlined here, however if we use
 276     //       slow unlocking, speed doesn't matter anyway and this solution is
 277     //       simpler and requires less duplicated code - additionally, the
 278     //       slow unlocking code is the same in either case which simplifies
 279     //       debugging
 280     __ br(Assembler::always, false, Assembler::pt, *slow_case->entry());
 281     __ delayed()->nop();
 282   }
 283   // done
 284   __ bind(*slow_case->continuation());
 285 }
 286 
 287 
 288 int LIR_Assembler::emit_exception_handler() {
 289   // if the last instruction is a call (typically to do a throw which
 290   // is coming at the end after block reordering) the return address
 291   // must still point into the code area in order to avoid assertion
 292   // failures when searching for the corresponding bci => add a nop
 293   // (was bug 5/14/1999 - gri)
 294   __ nop();
 295 
 296   // generate code for exception handler
 297   ciMethod* method = compilation()->method();
 298 
 299   address handler_base = __ start_a_stub(exception_handler_size());
 300 
 301   if (handler_base == NULL) {
 302     // not enough space left for the handler
 303     bailout("exception handler overflow");
 304     return -1;
 305   }
 306 
 307   int offset = code_offset();
 308 
 309   __ call(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id), relocInfo::runtime_call_type);
 310   __ delayed()->nop();
 311   __ should_not_reach_here();
 312   guarantee(code_offset() - offset <= exception_handler_size(), "overflow");
 313   __ end_a_stub();
 314 
 315   return offset;
 316 }
 317 
 318 
 319 // Emit the code to remove the frame from the stack in the exception
 320 // unwind path.
 321 int LIR_Assembler::emit_unwind_handler() {
 322 #ifndef PRODUCT
 323   if (CommentedAssembly) {
 324     _masm->block_comment("Unwind handler");
 325   }
 326 #endif
 327 
 328   int offset = code_offset();
 329 
 330   // Fetch the exception from TLS and clear out exception related thread state
 331   __ ld_ptr(G2_thread, in_bytes(JavaThread::exception_oop_offset()), O0);
 332   __ st_ptr(G0, G2_thread, in_bytes(JavaThread::exception_oop_offset()));
 333   __ st_ptr(G0, G2_thread, in_bytes(JavaThread::exception_pc_offset()));
 334 
 335   __ bind(_unwind_handler_entry);
 336   __ verify_not_null_oop(O0);
 337   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 338     __ mov(O0, I0);  // Preserve the exception
 339   }
 340 
 341   // Preform needed unlocking
 342   MonitorExitStub* stub = NULL;
 343   if (method()->is_synchronized()) {
 344     monitor_address(0, FrameMap::I1_opr);
 345     stub = new MonitorExitStub(FrameMap::I1_opr, true, 0);
 346     __ unlock_object(I3, I2, I1, *stub->entry());
 347     __ bind(*stub->continuation());
 348   }
 349 
 350   if (compilation()->env()->dtrace_method_probes()) {
 351     __ mov(G2_thread, O0);
 352     __ save_thread(I1); // need to preserve thread in G2 across
 353                         // runtime call
 354     metadata2reg(method()->constant_encoding(), O1);
 355     __ call(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), relocInfo::runtime_call_type);
 356     __ delayed()->nop();
 357     __ restore_thread(I1);
 358   }
 359 
 360   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 361     __ mov(I0, O0);  // Restore the exception
 362   }
 363 
 364   // dispatch to the unwind logic
 365   __ call(Runtime1::entry_for(Runtime1::unwind_exception_id), relocInfo::runtime_call_type);
 366   __ delayed()->nop();
 367 
 368   // Emit the slow path assembly
 369   if (stub != NULL) {
 370     stub->emit_code(this);
 371   }
 372 
 373   return offset;
 374 }
 375 
 376 
 377 int LIR_Assembler::emit_deopt_handler() {
 378   // if the last instruction is a call (typically to do a throw which
 379   // is coming at the end after block reordering) the return address
 380   // must still point into the code area in order to avoid assertion
 381   // failures when searching for the corresponding bci => add a nop
 382   // (was bug 5/14/1999 - gri)
 383   __ nop();
 384 
 385   // generate code for deopt handler
 386   ciMethod* method = compilation()->method();
 387   address handler_base = __ start_a_stub(deopt_handler_size());
 388   if (handler_base == NULL) {
 389     // not enough space left for the handler
 390     bailout("deopt handler overflow");
 391     return -1;
 392   }
 393 
 394   int offset = code_offset();
 395   AddressLiteral deopt_blob(SharedRuntime::deopt_blob()->unpack());
 396   __ JUMP(deopt_blob, G3_scratch, 0); // sethi;jmp
 397   __ delayed()->nop();
 398   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 399   __ end_a_stub();
 400 
 401   return offset;
 402 }
 403 
 404 
 405 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 406   if (o == NULL) {
 407     __ set(NULL_WORD, reg);
 408   } else {
 409 #ifdef ASSERT
 410     {
 411       ThreadInVMfromNative tiv(JavaThread::current());
 412       assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(o)), "should be real oop");
 413     }
 414 #endif
 415     int oop_index = __ oop_recorder()->find_index(o);
 416     RelocationHolder rspec = oop_Relocation::spec(oop_index);
 417     __ set(NULL_WORD, reg, rspec); // Will be set when the nmethod is created
 418   }
 419 }
 420 
 421 
 422 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 423   // Allocate a new index in table to hold the object once it's been patched
 424   int oop_index = __ oop_recorder()->allocate_oop_index(NULL);
 425   PatchingStub* patch = new PatchingStub(_masm, patching_id(info), oop_index);
 426 
 427   AddressLiteral addrlit(NULL, oop_Relocation::spec(oop_index));
 428   assert(addrlit.rspec().type() == relocInfo::oop_type, "must be an oop reloc");
 429   // It may not seem necessary to use a sethi/add pair to load a NULL into dest, but the
 430   // NULL will be dynamically patched later and the patched value may be large.  We must
 431   // therefore generate the sethi/add as a placeholders
 432   __ patchable_set(addrlit, reg);
 433 
 434   patching_epilog(patch, lir_patch_normal, reg, info);
 435 }
 436 
 437 
 438 void LIR_Assembler::metadata2reg(Metadata* o, Register reg) {
 439   __ set_metadata_constant(o, reg);
 440 }
 441 
 442 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo *info) {
 443   // Allocate a new index in table to hold the klass once it's been patched
 444   int index = __ oop_recorder()->allocate_metadata_index(NULL);
 445   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index);
 446   AddressLiteral addrlit(NULL, metadata_Relocation::spec(index));
 447   assert(addrlit.rspec().type() == relocInfo::metadata_type, "must be an metadata reloc");
 448   // It may not seem necessary to use a sethi/add pair to load a NULL into dest, but the
 449   // NULL will be dynamically patched later and the patched value may be large.  We must
 450   // therefore generate the sethi/add as a placeholders
 451   __ patchable_set(addrlit, reg);
 452 
 453   patching_epilog(patch, lir_patch_normal, reg, info);
 454 }
 455 
 456 void LIR_Assembler::emit_op3(LIR_Op3* op) {
 457   switch (op->code()) {
 458     case lir_idiv:
 459     case lir_irem:  // Both idiv & irem are handled after the switch (below).
 460       break;
 461     case lir_fmaf:
 462       __ fmadd(FloatRegisterImpl::S,
 463                op->in_opr1()->as_float_reg(),
 464                op->in_opr2()->as_float_reg(),
 465                op->in_opr3()->as_float_reg(),
 466                op->result_opr()->as_float_reg());
 467       return;
 468     case lir_fmad:
 469       __ fmadd(FloatRegisterImpl::D,
 470                op->in_opr1()->as_double_reg(),
 471                op->in_opr2()->as_double_reg(),
 472                op->in_opr3()->as_double_reg(),
 473                op->result_opr()->as_double_reg());
 474       return;
 475     default:
 476       ShouldNotReachHere();
 477       break;
 478   }
 479 
 480   // Handle idiv & irem:
 481 
 482   Register Rdividend = op->in_opr1()->as_register();
 483   Register Rdivisor  = noreg;
 484   Register Rscratch  = op->in_opr3()->as_register();
 485   Register Rresult   = op->result_opr()->as_register();
 486   int divisor = -1;
 487 
 488   if (op->in_opr2()->is_register()) {
 489     Rdivisor = op->in_opr2()->as_register();
 490   } else {
 491     divisor = op->in_opr2()->as_constant_ptr()->as_jint();
 492     assert(Assembler::is_simm13(divisor), "can only handle simm13");
 493   }
 494 
 495   assert(Rdividend != Rscratch, "");
 496   assert(Rdivisor  != Rscratch, "");
 497   assert(op->code() == lir_idiv || op->code() == lir_irem, "Must be irem or idiv");
 498 
 499   if (Rdivisor == noreg && is_power_of_2(divisor)) {
 500     // convert division by a power of two into some shifts and logical operations
 501     if (op->code() == lir_idiv) {
 502       if (divisor == 2) {
 503         __ srl(Rdividend, 31, Rscratch);
 504       } else {
 505         __ sra(Rdividend, 31, Rscratch);
 506         __ and3(Rscratch, divisor - 1, Rscratch);
 507       }
 508       __ add(Rdividend, Rscratch, Rscratch);
 509       __ sra(Rscratch, log2_int(divisor), Rresult);
 510       return;
 511     } else {
 512       if (divisor == 2) {
 513         __ srl(Rdividend, 31, Rscratch);
 514       } else {
 515         __ sra(Rdividend, 31, Rscratch);
 516         __ and3(Rscratch, divisor - 1,Rscratch);
 517       }
 518       __ add(Rdividend, Rscratch, Rscratch);
 519       __ andn(Rscratch, divisor - 1,Rscratch);
 520       __ sub(Rdividend, Rscratch, Rresult);
 521       return;
 522     }
 523   }
 524 
 525   __ sra(Rdividend, 31, Rscratch);
 526   __ wry(Rscratch);
 527 
 528   add_debug_info_for_div0_here(op->info());
 529 
 530   if (Rdivisor != noreg) {
 531     __ sdivcc(Rdividend, Rdivisor, (op->code() == lir_idiv ? Rresult : Rscratch));
 532   } else {
 533     assert(Assembler::is_simm13(divisor), "can only handle simm13");
 534     __ sdivcc(Rdividend, divisor, (op->code() == lir_idiv ? Rresult : Rscratch));
 535   }
 536 
 537   Label skip;
 538   __ br(Assembler::overflowSet, true, Assembler::pn, skip);
 539   __ delayed()->Assembler::sethi(0x80000000, (op->code() == lir_idiv ? Rresult : Rscratch));
 540   __ bind(skip);
 541 
 542   if (op->code() == lir_irem) {
 543     if (Rdivisor != noreg) {
 544       __ smul(Rscratch, Rdivisor, Rscratch);
 545     } else {
 546       __ smul(Rscratch, divisor, Rscratch);
 547     }
 548     __ sub(Rdividend, Rscratch, Rresult);
 549   }
 550 }
 551 
 552 
 553 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
 554 #ifdef ASSERT
 555   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
 556   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
 557   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
 558 #endif
 559   assert(op->info() == NULL, "shouldn't have CodeEmitInfo");
 560 
 561   if (op->cond() == lir_cond_always) {
 562     __ br(Assembler::always, false, Assembler::pt, *(op->label()));
 563   } else if (op->code() == lir_cond_float_branch) {
 564     assert(op->ublock() != NULL, "must have unordered successor");
 565     bool is_unordered = (op->ublock() == op->block());
 566     Assembler::Condition acond;
 567     switch (op->cond()) {
 568       case lir_cond_equal:         acond = Assembler::f_equal;    break;
 569       case lir_cond_notEqual:      acond = Assembler::f_notEqual; break;
 570       case lir_cond_less:          acond = (is_unordered ? Assembler::f_unorderedOrLess          : Assembler::f_less);           break;
 571       case lir_cond_greater:       acond = (is_unordered ? Assembler::f_unorderedOrGreater       : Assembler::f_greater);        break;
 572       case lir_cond_lessEqual:     acond = (is_unordered ? Assembler::f_unorderedOrLessOrEqual   : Assembler::f_lessOrEqual);    break;
 573       case lir_cond_greaterEqual:  acond = (is_unordered ? Assembler::f_unorderedOrGreaterOrEqual: Assembler::f_greaterOrEqual); break;
 574       default :                         ShouldNotReachHere();
 575     }
 576     __ fb( acond, false, Assembler::pn, *(op->label()));
 577   } else {
 578     assert (op->code() == lir_branch, "just checking");
 579 
 580     Assembler::Condition acond;
 581     switch (op->cond()) {
 582       case lir_cond_equal:        acond = Assembler::equal;                break;
 583       case lir_cond_notEqual:     acond = Assembler::notEqual;             break;
 584       case lir_cond_less:         acond = Assembler::less;                 break;
 585       case lir_cond_lessEqual:    acond = Assembler::lessEqual;            break;
 586       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;         break;
 587       case lir_cond_greater:      acond = Assembler::greater;              break;
 588       case lir_cond_aboveEqual:   acond = Assembler::greaterEqualUnsigned; break;
 589       case lir_cond_belowEqual:   acond = Assembler::lessEqualUnsigned;    break;
 590       default:                         ShouldNotReachHere();
 591     };
 592 
 593     // sparc has different condition codes for testing 32-bit
 594     // vs. 64-bit values.  We could always test xcc is we could
 595     // guarantee that 32-bit loads always sign extended but that isn't
 596     // true and since sign extension isn't free, it would impose a
 597     // slight cost.
 598     if  (op->type() == T_INT) {
 599       __ br(acond, false, Assembler::pn, *(op->label()));
 600     } else
 601       __ brx(acond, false, Assembler::pn, *(op->label()));
 602   }
 603   // The peephole pass fills the delay slot
 604 }
 605 
 606 
 607 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
 608   Bytecodes::Code code = op->bytecode();
 609   LIR_Opr dst = op->result_opr();
 610 
 611   switch(code) {
 612     case Bytecodes::_i2l: {
 613       Register rlo  = dst->as_register_lo();
 614       Register rhi  = dst->as_register_hi();
 615       Register rval = op->in_opr()->as_register();
 616       __ sra(rval, 0, rlo);
 617       break;
 618     }
 619     case Bytecodes::_i2d:
 620     case Bytecodes::_i2f: {
 621       bool is_double = (code == Bytecodes::_i2d);
 622       FloatRegister rdst = is_double ? dst->as_double_reg() : dst->as_float_reg();
 623       FloatRegisterImpl::Width w = is_double ? FloatRegisterImpl::D : FloatRegisterImpl::S;
 624       FloatRegister rsrc = op->in_opr()->as_float_reg();
 625       if (rsrc != rdst) {
 626         __ fmov(FloatRegisterImpl::S, rsrc, rdst);
 627       }
 628       __ fitof(w, rdst, rdst);
 629       break;
 630     }
 631     case Bytecodes::_f2i:{
 632       FloatRegister rsrc = op->in_opr()->as_float_reg();
 633       Address       addr = frame_map()->address_for_slot(dst->single_stack_ix());
 634       Label L;
 635       // result must be 0 if value is NaN; test by comparing value to itself
 636       __ fcmp(FloatRegisterImpl::S, Assembler::fcc0, rsrc, rsrc);
 637       __ fb(Assembler::f_unordered, true, Assembler::pn, L);
 638       __ delayed()->st(G0, addr); // annuled if contents of rsrc is not NaN
 639       __ ftoi(FloatRegisterImpl::S, rsrc, rsrc);
 640       // move integer result from float register to int register
 641       __ stf(FloatRegisterImpl::S, rsrc, addr.base(), addr.disp());
 642       __ bind (L);
 643       break;
 644     }
 645     case Bytecodes::_l2i: {
 646       Register rlo  = op->in_opr()->as_register_lo();
 647       Register rhi  = op->in_opr()->as_register_hi();
 648       Register rdst = dst->as_register();
 649       __ sra(rlo, 0, rdst);
 650       break;
 651     }
 652     case Bytecodes::_d2f:
 653     case Bytecodes::_f2d: {
 654       bool is_double = (code == Bytecodes::_f2d);
 655       assert((!is_double && dst->is_single_fpu()) || (is_double && dst->is_double_fpu()), "check");
 656       LIR_Opr val = op->in_opr();
 657       FloatRegister rval = (code == Bytecodes::_d2f) ? val->as_double_reg() : val->as_float_reg();
 658       FloatRegister rdst = is_double ? dst->as_double_reg() : dst->as_float_reg();
 659       FloatRegisterImpl::Width vw = is_double ? FloatRegisterImpl::S : FloatRegisterImpl::D;
 660       FloatRegisterImpl::Width dw = is_double ? FloatRegisterImpl::D : FloatRegisterImpl::S;
 661       __ ftof(vw, dw, rval, rdst);
 662       break;
 663     }
 664     case Bytecodes::_i2s:
 665     case Bytecodes::_i2b: {
 666       Register rval = op->in_opr()->as_register();
 667       Register rdst = dst->as_register();
 668       int shift = (code == Bytecodes::_i2b) ? (BitsPerInt - T_BYTE_aelem_bytes * BitsPerByte) : (BitsPerInt - BitsPerShort);
 669       __ sll (rval, shift, rdst);
 670       __ sra (rdst, shift, rdst);
 671       break;
 672     }
 673     case Bytecodes::_i2c: {
 674       Register rval = op->in_opr()->as_register();
 675       Register rdst = dst->as_register();
 676       int shift = BitsPerInt - T_CHAR_aelem_bytes * BitsPerByte;
 677       __ sll (rval, shift, rdst);
 678       __ srl (rdst, shift, rdst);
 679       break;
 680     }
 681 
 682     default: ShouldNotReachHere();
 683   }
 684 }
 685 
 686 
 687 void LIR_Assembler::align_call(LIR_Code) {
 688   // do nothing since all instructions are word aligned on sparc
 689 }
 690 
 691 
 692 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
 693   __ call(op->addr(), rtype);
 694   // The peephole pass fills the delay slot, add_call_info is done in
 695   // LIR_Assembler::emit_delay.
 696 }
 697 
 698 
 699 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
 700   __ ic_call(op->addr(), false);
 701   // The peephole pass fills the delay slot, add_call_info is done in
 702   // LIR_Assembler::emit_delay.
 703 }
 704 
 705 
 706 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
 707   add_debug_info_for_null_check_here(op->info());
 708   __ load_klass(O0, G3_scratch);
 709   if (Assembler::is_simm13(op->vtable_offset())) {
 710     __ ld_ptr(G3_scratch, op->vtable_offset(), G5_method);
 711   } else {
 712     // This will generate 2 instructions
 713     __ set(op->vtable_offset(), G5_method);
 714     // ld_ptr, set_hi, set
 715     __ ld_ptr(G3_scratch, G5_method, G5_method);
 716   }
 717   __ ld_ptr(G5_method, Method::from_compiled_offset(), G3_scratch);
 718   __ callr(G3_scratch, G0);
 719   // the peephole pass fills the delay slot
 720 }
 721 
 722 int LIR_Assembler::store(LIR_Opr from_reg, Register base, int offset, BasicType type, bool wide, bool unaligned) {
 723   int store_offset;
 724   if (!Assembler::is_simm13(offset + (type == T_LONG) ? wordSize : 0)) {
 725     assert(base != O7, "destroying register");
 726     assert(!unaligned, "can't handle this");
 727     // for offsets larger than a simm13 we setup the offset in O7
 728     __ set(offset, O7);
 729     store_offset = store(from_reg, base, O7, type, wide);
 730   } else {
 731     if (type == T_ARRAY || type == T_OBJECT) {
 732       __ verify_oop(from_reg->as_register());
 733     }
 734     store_offset = code_offset();
 735     switch (type) {
 736       case T_BOOLEAN: // fall through
 737       case T_BYTE  : __ stb(from_reg->as_register(), base, offset); break;
 738       case T_CHAR  : __ sth(from_reg->as_register(), base, offset); break;
 739       case T_SHORT : __ sth(from_reg->as_register(), base, offset); break;
 740       case T_INT   : __ stw(from_reg->as_register(), base, offset); break;
 741       case T_LONG  :
 742         if (unaligned || PatchALot) {
 743           // Don't use O7 here because it may be equal to 'base' (see LIR_Assembler::reg2mem)
 744           assert(G3_scratch != base, "can't handle this");
 745           assert(G3_scratch != from_reg->as_register_lo(), "can't handle this");
 746           __ srax(from_reg->as_register_lo(), 32, G3_scratch);
 747           __ stw(from_reg->as_register_lo(), base, offset + lo_word_offset_in_bytes);
 748           __ stw(G3_scratch,                 base, offset + hi_word_offset_in_bytes);
 749         } else {
 750           __ stx(from_reg->as_register_lo(), base, offset);
 751         }
 752         break;
 753       case T_ADDRESS:
 754       case T_METADATA:
 755         __ st_ptr(from_reg->as_register(), base, offset);
 756         break;
 757       case T_ARRAY : // fall through
 758       case T_OBJECT:
 759         {
 760           if (UseCompressedOops && !wide) {
 761             __ encode_heap_oop(from_reg->as_register(), G3_scratch);
 762             store_offset = code_offset();
 763             __ stw(G3_scratch, base, offset);
 764           } else {
 765             __ st_ptr(from_reg->as_register(), base, offset);
 766           }
 767           break;
 768         }
 769 
 770       case T_FLOAT : __ stf(FloatRegisterImpl::S, from_reg->as_float_reg(), base, offset); break;
 771       case T_DOUBLE:
 772         {
 773           FloatRegister reg = from_reg->as_double_reg();
 774           // split unaligned stores
 775           if (unaligned || PatchALot) {
 776             assert(Assembler::is_simm13(offset + 4), "must be");
 777             __ stf(FloatRegisterImpl::S, reg->successor(), base, offset + 4);
 778             __ stf(FloatRegisterImpl::S, reg,              base, offset);
 779           } else {
 780             __ stf(FloatRegisterImpl::D, reg, base, offset);
 781           }
 782           break;
 783         }
 784       default      : ShouldNotReachHere();
 785     }
 786   }
 787   return store_offset;
 788 }
 789 
 790 
 791 int LIR_Assembler::store(LIR_Opr from_reg, Register base, Register disp, BasicType type, bool wide) {
 792   if (type == T_ARRAY || type == T_OBJECT) {
 793     __ verify_oop(from_reg->as_register());
 794   }
 795   int store_offset = code_offset();
 796   switch (type) {
 797     case T_BOOLEAN: // fall through
 798     case T_BYTE  : __ stb(from_reg->as_register(), base, disp); break;
 799     case T_CHAR  : __ sth(from_reg->as_register(), base, disp); break;
 800     case T_SHORT : __ sth(from_reg->as_register(), base, disp); break;
 801     case T_INT   : __ stw(from_reg->as_register(), base, disp); break;
 802     case T_LONG  :
 803       __ stx(from_reg->as_register_lo(), base, disp);
 804       break;
 805     case T_ADDRESS:
 806       __ st_ptr(from_reg->as_register(), base, disp);
 807       break;
 808     case T_ARRAY : // fall through
 809     case T_OBJECT:
 810       {
 811         if (UseCompressedOops && !wide) {
 812           __ encode_heap_oop(from_reg->as_register(), G3_scratch);
 813           store_offset = code_offset();
 814           __ stw(G3_scratch, base, disp);
 815         } else {
 816           __ st_ptr(from_reg->as_register(), base, disp);
 817         }
 818         break;
 819       }
 820     case T_FLOAT : __ stf(FloatRegisterImpl::S, from_reg->as_float_reg(), base, disp); break;
 821     case T_DOUBLE: __ stf(FloatRegisterImpl::D, from_reg->as_double_reg(), base, disp); break;
 822     default      : ShouldNotReachHere();
 823   }
 824   return store_offset;
 825 }
 826 
 827 
 828 int LIR_Assembler::load(Register base, int offset, LIR_Opr to_reg, BasicType type, bool wide, bool unaligned) {
 829   int load_offset;
 830   if (!Assembler::is_simm13(offset + (type == T_LONG) ? wordSize : 0)) {
 831     assert(base != O7, "destroying register");
 832     assert(!unaligned, "can't handle this");
 833     // for offsets larger than a simm13 we setup the offset in O7
 834     __ set(offset, O7);
 835     load_offset = load(base, O7, to_reg, type, wide);
 836   } else {
 837     load_offset = code_offset();
 838     switch(type) {
 839       case T_BOOLEAN: // fall through
 840       case T_BYTE  : __ ldsb(base, offset, to_reg->as_register()); break;
 841       case T_CHAR  : __ lduh(base, offset, to_reg->as_register()); break;
 842       case T_SHORT : __ ldsh(base, offset, to_reg->as_register()); break;
 843       case T_INT   : __ ld(base, offset, to_reg->as_register()); break;
 844       case T_LONG  :
 845         if (!unaligned && !PatchALot) {
 846           __ ldx(base, offset, to_reg->as_register_lo());
 847         } else {
 848           assert(base != to_reg->as_register_lo(), "can't handle this");
 849           assert(O7 != to_reg->as_register_lo(), "can't handle this");
 850           __ ld(base, offset + hi_word_offset_in_bytes, to_reg->as_register_lo());
 851           __ lduw(base, offset + lo_word_offset_in_bytes, O7); // in case O7 is base or offset, use it last
 852           __ sllx(to_reg->as_register_lo(), 32, to_reg->as_register_lo());
 853           __ or3(to_reg->as_register_lo(), O7, to_reg->as_register_lo());
 854         }
 855         break;
 856       case T_METADATA:  __ ld_ptr(base, offset, to_reg->as_register()); break;
 857       case T_ADDRESS:
 858         if (offset == oopDesc::klass_offset_in_bytes() && UseCompressedClassPointers) {
 859           __ lduw(base, offset, to_reg->as_register());
 860           __ decode_klass_not_null(to_reg->as_register());
 861         } else
 862         {
 863           __ ld_ptr(base, offset, to_reg->as_register());
 864         }
 865         break;
 866       case T_ARRAY : // fall through
 867       case T_OBJECT:
 868         {
 869           if (UseCompressedOops && !wide) {
 870             __ lduw(base, offset, to_reg->as_register());
 871             __ decode_heap_oop(to_reg->as_register());
 872           } else {
 873             __ ld_ptr(base, offset, to_reg->as_register());
 874           }
 875           break;
 876         }
 877       case T_FLOAT:  __ ldf(FloatRegisterImpl::S, base, offset, to_reg->as_float_reg()); break;
 878       case T_DOUBLE:
 879         {
 880           FloatRegister reg = to_reg->as_double_reg();
 881           // split unaligned loads
 882           if (unaligned || PatchALot) {
 883             __ ldf(FloatRegisterImpl::S, base, offset + 4, reg->successor());
 884             __ ldf(FloatRegisterImpl::S, base, offset,     reg);
 885           } else {
 886             __ ldf(FloatRegisterImpl::D, base, offset, to_reg->as_double_reg());
 887           }
 888           break;
 889         }
 890       default      : ShouldNotReachHere();
 891     }
 892     if (type == T_ARRAY || type == T_OBJECT) {
 893       __ verify_oop(to_reg->as_register());
 894     }
 895   }
 896   return load_offset;
 897 }
 898 
 899 
 900 int LIR_Assembler::load(Register base, Register disp, LIR_Opr to_reg, BasicType type, bool wide) {
 901   int load_offset = code_offset();
 902   switch(type) {
 903     case T_BOOLEAN: // fall through
 904     case T_BYTE  :  __ ldsb(base, disp, to_reg->as_register()); break;
 905     case T_CHAR  :  __ lduh(base, disp, to_reg->as_register()); break;
 906     case T_SHORT :  __ ldsh(base, disp, to_reg->as_register()); break;
 907     case T_INT   :  __ ld(base, disp, to_reg->as_register()); break;
 908     case T_ADDRESS: __ ld_ptr(base, disp, to_reg->as_register()); break;
 909     case T_ARRAY : // fall through
 910     case T_OBJECT:
 911       {
 912           if (UseCompressedOops && !wide) {
 913             __ lduw(base, disp, to_reg->as_register());
 914             __ decode_heap_oop(to_reg->as_register());
 915           } else {
 916             __ ld_ptr(base, disp, to_reg->as_register());
 917           }
 918           break;
 919       }
 920     case T_FLOAT:  __ ldf(FloatRegisterImpl::S, base, disp, to_reg->as_float_reg()); break;
 921     case T_DOUBLE: __ ldf(FloatRegisterImpl::D, base, disp, to_reg->as_double_reg()); break;
 922     case T_LONG  :
 923       __ ldx(base, disp, to_reg->as_register_lo());
 924       break;
 925     default      : ShouldNotReachHere();
 926   }
 927   if (type == T_ARRAY || type == T_OBJECT) {
 928     __ verify_oop(to_reg->as_register());
 929   }
 930   return load_offset;
 931 }
 932 
 933 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
 934   LIR_Const* c = src->as_constant_ptr();
 935   switch (c->type()) {
 936     case T_INT:
 937     case T_FLOAT: {
 938       Register src_reg = O7;
 939       int value = c->as_jint_bits();
 940       if (value == 0) {
 941         src_reg = G0;
 942       } else {
 943         __ set(value, O7);
 944       }
 945       Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
 946       __ stw(src_reg, addr.base(), addr.disp());
 947       break;
 948     }
 949     case T_ADDRESS: {
 950       Register src_reg = O7;
 951       int value = c->as_jint_bits();
 952       if (value == 0) {
 953         src_reg = G0;
 954       } else {
 955         __ set(value, O7);
 956       }
 957       Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
 958       __ st_ptr(src_reg, addr.base(), addr.disp());
 959       break;
 960     }
 961     case T_OBJECT: {
 962       Register src_reg = O7;
 963       jobject2reg(c->as_jobject(), src_reg);
 964       Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
 965       __ st_ptr(src_reg, addr.base(), addr.disp());
 966       break;
 967     }
 968     case T_LONG:
 969     case T_DOUBLE: {
 970       Address addr = frame_map()->address_for_double_slot(dest->double_stack_ix());
 971 
 972       Register tmp = O7;
 973       int value_lo = c->as_jint_lo_bits();
 974       if (value_lo == 0) {
 975         tmp = G0;
 976       } else {
 977         __ set(value_lo, O7);
 978       }
 979       __ stw(tmp, addr.base(), addr.disp() + lo_word_offset_in_bytes);
 980       int value_hi = c->as_jint_hi_bits();
 981       if (value_hi == 0) {
 982         tmp = G0;
 983       } else {
 984         __ set(value_hi, O7);
 985       }
 986       __ stw(tmp, addr.base(), addr.disp() + hi_word_offset_in_bytes);
 987       break;
 988     }
 989     default:
 990       Unimplemented();
 991   }
 992 }
 993 
 994 
 995 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
 996   LIR_Const* c = src->as_constant_ptr();
 997   LIR_Address* addr     = dest->as_address_ptr();
 998   Register base = addr->base()->as_pointer_register();
 999   int offset = -1;
1000 
1001   switch (c->type()) {
1002     case T_FLOAT: type = T_INT; // Float constants are stored by int store instructions.
1003     case T_INT:
1004     case T_ADDRESS: {
1005       LIR_Opr tmp = FrameMap::O7_opr;
1006       int value = c->as_jint_bits();
1007       if (value == 0) {
1008         tmp = FrameMap::G0_opr;
1009       } else if (Assembler::is_simm13(value)) {
1010         __ set(value, O7);
1011       }
1012       if (addr->index()->is_valid()) {
1013         assert(addr->disp() == 0, "must be zero");
1014         offset = store(tmp, base, addr->index()->as_pointer_register(), type, wide);
1015       } else {
1016         assert(Assembler::is_simm13(addr->disp()), "can't handle larger addresses");
1017         offset = store(tmp, base, addr->disp(), type, wide, false);
1018       }
1019       break;
1020     }
1021     case T_LONG:
1022     case T_DOUBLE: {
1023       assert(!addr->index()->is_valid(), "can't handle reg reg address here");
1024       assert(Assembler::is_simm13(addr->disp()) &&
1025              Assembler::is_simm13(addr->disp() + 4), "can't handle larger addresses");
1026 
1027       LIR_Opr tmp = FrameMap::O7_opr;
1028       int value_lo = c->as_jint_lo_bits();
1029       if (value_lo == 0) {
1030         tmp = FrameMap::G0_opr;
1031       } else {
1032         __ set(value_lo, O7);
1033       }
1034       offset = store(tmp, base, addr->disp() + lo_word_offset_in_bytes, T_INT, wide, false);
1035       int value_hi = c->as_jint_hi_bits();
1036       if (value_hi == 0) {
1037         tmp = FrameMap::G0_opr;
1038       } else {
1039         __ set(value_hi, O7);
1040       }
1041       store(tmp, base, addr->disp() + hi_word_offset_in_bytes, T_INT, wide, false);
1042       break;
1043     }
1044     case T_OBJECT: {
1045       jobject obj = c->as_jobject();
1046       LIR_Opr tmp;
1047       if (obj == NULL) {
1048         tmp = FrameMap::G0_opr;
1049       } else {
1050         tmp = FrameMap::O7_opr;
1051         jobject2reg(c->as_jobject(), O7);
1052       }
1053       // handle either reg+reg or reg+disp address
1054       if (addr->index()->is_valid()) {
1055         assert(addr->disp() == 0, "must be zero");
1056         offset = store(tmp, base, addr->index()->as_pointer_register(), type, wide);
1057       } else {
1058         assert(Assembler::is_simm13(addr->disp()), "can't handle larger addresses");
1059         offset = store(tmp, base, addr->disp(), type, wide, false);
1060       }
1061 
1062       break;
1063     }
1064     default:
1065       Unimplemented();
1066   }
1067   if (info != NULL) {
1068     assert(offset != -1, "offset should've been set");
1069     add_debug_info_for_null_check(offset, info);
1070   }
1071 }
1072 
1073 
1074 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
1075   LIR_Const* c = src->as_constant_ptr();
1076   LIR_Opr to_reg = dest;
1077 
1078   switch (c->type()) {
1079     case T_INT:
1080     case T_ADDRESS:
1081       {
1082         jint con = c->as_jint();
1083         if (to_reg->is_single_cpu()) {
1084           assert(patch_code == lir_patch_none, "no patching handled here");
1085           __ set(con, to_reg->as_register());
1086         } else {
1087           ShouldNotReachHere();
1088           assert(to_reg->is_single_fpu(), "wrong register kind");
1089 
1090           __ set(con, O7);
1091           Address temp_slot(SP, (frame::register_save_words * wordSize) + STACK_BIAS);
1092           __ st(O7, temp_slot);
1093           __ ldf(FloatRegisterImpl::S, temp_slot, to_reg->as_float_reg());
1094         }
1095       }
1096       break;
1097 
1098     case T_LONG:
1099       {
1100         jlong con = c->as_jlong();
1101 
1102         if (to_reg->is_double_cpu()) {
1103           __ set(con,  to_reg->as_register_lo());
1104         } else if (to_reg->is_single_cpu()) {
1105           __ set(con, to_reg->as_register());
1106         } else {
1107           ShouldNotReachHere();
1108           assert(to_reg->is_double_fpu(), "wrong register kind");
1109           Address temp_slot_lo(SP, ((frame::register_save_words  ) * wordSize) + STACK_BIAS);
1110           Address temp_slot_hi(SP, ((frame::register_save_words) * wordSize) + (longSize/2) + STACK_BIAS);
1111           __ set(low(con),  O7);
1112           __ st(O7, temp_slot_lo);
1113           __ set(high(con), O7);
1114           __ st(O7, temp_slot_hi);
1115           __ ldf(FloatRegisterImpl::D, temp_slot_lo, to_reg->as_double_reg());
1116         }
1117       }
1118       break;
1119 
1120     case T_OBJECT:
1121       {
1122         if (patch_code == lir_patch_none) {
1123           jobject2reg(c->as_jobject(), to_reg->as_register());
1124         } else {
1125           jobject2reg_with_patching(to_reg->as_register(), info);
1126         }
1127       }
1128       break;
1129 
1130     case T_METADATA:
1131       {
1132         if (patch_code == lir_patch_none) {
1133           metadata2reg(c->as_metadata(), to_reg->as_register());
1134         } else {
1135           klass2reg_with_patching(to_reg->as_register(), info);
1136         }
1137       }
1138       break;
1139 
1140     case T_FLOAT:
1141       {
1142         address const_addr = __ float_constant(c->as_jfloat());
1143         if (const_addr == NULL) {
1144           bailout("const section overflow");
1145           break;
1146         }
1147         RelocationHolder rspec = internal_word_Relocation::spec(const_addr);
1148         AddressLiteral const_addrlit(const_addr, rspec);
1149         if (to_reg->is_single_fpu()) {
1150           __ patchable_sethi(const_addrlit, O7);
1151           __ relocate(rspec);
1152           __ ldf(FloatRegisterImpl::S, O7, const_addrlit.low10(), to_reg->as_float_reg());
1153 
1154         } else {
1155           assert(to_reg->is_single_cpu(), "Must be a cpu register.");
1156 
1157           __ set(const_addrlit, O7);
1158           __ ld(O7, 0, to_reg->as_register());
1159         }
1160       }
1161       break;
1162 
1163     case T_DOUBLE:
1164       {
1165         address const_addr = __ double_constant(c->as_jdouble());
1166         if (const_addr == NULL) {
1167           bailout("const section overflow");
1168           break;
1169         }
1170         RelocationHolder rspec = internal_word_Relocation::spec(const_addr);
1171 
1172         if (to_reg->is_double_fpu()) {
1173           AddressLiteral const_addrlit(const_addr, rspec);
1174           __ patchable_sethi(const_addrlit, O7);
1175           __ relocate(rspec);
1176           __ ldf (FloatRegisterImpl::D, O7, const_addrlit.low10(), to_reg->as_double_reg());
1177         } else {
1178           assert(to_reg->is_double_cpu(), "Must be a long register.");
1179           __ set(jlong_cast(c->as_jdouble()), to_reg->as_register_lo());
1180         }
1181 
1182       }
1183       break;
1184 
1185     default:
1186       ShouldNotReachHere();
1187   }
1188 }
1189 
1190 Address LIR_Assembler::as_Address(LIR_Address* addr) {
1191   Register reg = addr->base()->as_pointer_register();
1192   LIR_Opr index = addr->index();
1193   if (index->is_illegal()) {
1194     return Address(reg, addr->disp());
1195   } else {
1196     assert (addr->disp() == 0, "unsupported address mode");
1197     return Address(reg, index->as_pointer_register());
1198   }
1199 }
1200 
1201 
1202 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1203   switch (type) {
1204     case T_INT:
1205     case T_FLOAT: {
1206       Register tmp = O7;
1207       Address from = frame_map()->address_for_slot(src->single_stack_ix());
1208       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
1209       __ lduw(from.base(), from.disp(), tmp);
1210       __ stw(tmp, to.base(), to.disp());
1211       break;
1212     }
1213     case T_ADDRESS:
1214     case T_OBJECT: {
1215       Register tmp = O7;
1216       Address from = frame_map()->address_for_slot(src->single_stack_ix());
1217       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
1218       __ ld_ptr(from.base(), from.disp(), tmp);
1219       __ st_ptr(tmp, to.base(), to.disp());
1220       break;
1221     }
1222     case T_LONG:
1223     case T_DOUBLE: {
1224       Register tmp = O7;
1225       Address from = frame_map()->address_for_double_slot(src->double_stack_ix());
1226       Address to   = frame_map()->address_for_double_slot(dest->double_stack_ix());
1227       __ lduw(from.base(), from.disp(), tmp);
1228       __ stw(tmp, to.base(), to.disp());
1229       __ lduw(from.base(), from.disp() + 4, tmp);
1230       __ stw(tmp, to.base(), to.disp() + 4);
1231       break;
1232     }
1233 
1234     default:
1235       ShouldNotReachHere();
1236   }
1237 }
1238 
1239 
1240 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
1241   Address base = as_Address(addr);
1242   return Address(base.base(), base.disp() + hi_word_offset_in_bytes);
1243 }
1244 
1245 
1246 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
1247   Address base = as_Address(addr);
1248   return Address(base.base(), base.disp() + lo_word_offset_in_bytes);
1249 }
1250 
1251 
1252 void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type,
1253                             LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool unaligned) {
1254 
1255   assert(type != T_METADATA, "load of metadata ptr not supported");
1256   LIR_Address* addr = src_opr->as_address_ptr();
1257   LIR_Opr to_reg = dest;
1258 
1259   Register src = addr->base()->as_pointer_register();
1260   Register disp_reg = noreg;
1261   int disp_value = addr->disp();
1262   bool needs_patching = (patch_code != lir_patch_none);
1263 
1264   if (addr->base()->type() == T_OBJECT) {
1265     __ verify_oop(src);
1266   }
1267 
1268   PatchingStub* patch = NULL;
1269   if (needs_patching) {
1270     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1271     assert(!to_reg->is_double_cpu() ||
1272            patch_code == lir_patch_none ||
1273            patch_code == lir_patch_normal, "patching doesn't match register");
1274   }
1275 
1276   if (addr->index()->is_illegal()) {
1277     if (!Assembler::is_simm13(disp_value) && (!unaligned || Assembler::is_simm13(disp_value + 4))) {
1278       if (needs_patching) {
1279         __ patchable_set(0, O7);
1280       } else {
1281         __ set(disp_value, O7);
1282       }
1283       disp_reg = O7;
1284     }
1285   } else if (unaligned || PatchALot) {
1286     __ add(src, addr->index()->as_pointer_register(), O7);
1287     src = O7;
1288   } else {
1289     disp_reg = addr->index()->as_pointer_register();
1290     assert(disp_value == 0, "can't handle 3 operand addresses");
1291   }
1292 
1293   // remember the offset of the load.  The patching_epilog must be done
1294   // before the call to add_debug_info, otherwise the PcDescs don't get
1295   // entered in increasing order.
1296   int offset = code_offset();
1297 
1298   assert(disp_reg != noreg || Assembler::is_simm13(disp_value), "should have set this up");
1299   if (disp_reg == noreg) {
1300     offset = load(src, disp_value, to_reg, type, wide, unaligned);
1301   } else {
1302     assert(!unaligned, "can't handle this");
1303     offset = load(src, disp_reg, to_reg, type, wide);
1304   }
1305 
1306   if (patch != NULL) {
1307     patching_epilog(patch, patch_code, src, info);
1308   }
1309   if (info != NULL) add_debug_info_for_null_check(offset, info);
1310 }
1311 
1312 
1313 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1314   Address addr;
1315   if (src->is_single_word()) {
1316     addr = frame_map()->address_for_slot(src->single_stack_ix());
1317   } else if (src->is_double_word())  {
1318     addr = frame_map()->address_for_double_slot(src->double_stack_ix());
1319   }
1320 
1321   bool unaligned = (addr.disp() - STACK_BIAS) % 8 != 0;
1322   load(addr.base(), addr.disp(), dest, dest->type(), true /*wide*/, unaligned);
1323 }
1324 
1325 
1326 void LIR_Assembler::reg2stack(LIR_Opr from_reg, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
1327   Address addr;
1328   if (dest->is_single_word()) {
1329     addr = frame_map()->address_for_slot(dest->single_stack_ix());
1330   } else if (dest->is_double_word())  {
1331     addr = frame_map()->address_for_slot(dest->double_stack_ix());
1332   }
1333   bool unaligned = (addr.disp() - STACK_BIAS) % 8 != 0;
1334   store(from_reg, addr.base(), addr.disp(), from_reg->type(), true /*wide*/, unaligned);
1335 }
1336 
1337 
1338 void LIR_Assembler::reg2reg(LIR_Opr from_reg, LIR_Opr to_reg) {
1339   if (from_reg->is_float_kind() && to_reg->is_float_kind()) {
1340     if (from_reg->is_double_fpu()) {
1341       // double to double moves
1342       assert(to_reg->is_double_fpu(), "should match");
1343       __ fmov(FloatRegisterImpl::D, from_reg->as_double_reg(), to_reg->as_double_reg());
1344     } else {
1345       // float to float moves
1346       assert(to_reg->is_single_fpu(), "should match");
1347       __ fmov(FloatRegisterImpl::S, from_reg->as_float_reg(), to_reg->as_float_reg());
1348     }
1349   } else if (!from_reg->is_float_kind() && !to_reg->is_float_kind()) {
1350     if (from_reg->is_double_cpu()) {
1351       __ mov(from_reg->as_pointer_register(), to_reg->as_pointer_register());
1352     } else if (to_reg->is_double_cpu()) {
1353       // int to int moves
1354       __ mov(from_reg->as_register(), to_reg->as_register_lo());
1355     } else {
1356       // int to int moves
1357       __ mov(from_reg->as_register(), to_reg->as_register());
1358     }
1359   } else {
1360     ShouldNotReachHere();
1361   }
1362   if (to_reg->type() == T_OBJECT || to_reg->type() == T_ARRAY) {
1363     __ verify_oop(to_reg->as_register());
1364   }
1365 }
1366 
1367 void LIR_Assembler::reg2mem(LIR_Opr from_reg, LIR_Opr dest, BasicType type,
1368                             LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack,
1369                             bool wide, bool unaligned) {
1370   assert(type != T_METADATA, "store of metadata ptr not supported");
1371   LIR_Address* addr = dest->as_address_ptr();
1372 
1373   Register src = addr->base()->as_pointer_register();
1374   Register disp_reg = noreg;
1375   int disp_value = addr->disp();
1376   bool needs_patching = (patch_code != lir_patch_none);
1377 
1378   if (addr->base()->is_oop_register()) {
1379     __ verify_oop(src);
1380   }
1381 
1382   PatchingStub* patch = NULL;
1383   if (needs_patching) {
1384     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1385     assert(!from_reg->is_double_cpu() ||
1386            patch_code == lir_patch_none ||
1387            patch_code == lir_patch_normal, "patching doesn't match register");
1388   }
1389 
1390   if (addr->index()->is_illegal()) {
1391     if (!Assembler::is_simm13(disp_value) && (!unaligned || Assembler::is_simm13(disp_value + 4))) {
1392       if (needs_patching) {
1393         __ patchable_set(0, O7);
1394       } else {
1395         __ set(disp_value, O7);
1396       }
1397       disp_reg = O7;
1398     }
1399   } else if (unaligned || PatchALot) {
1400     __ add(src, addr->index()->as_pointer_register(), O7);
1401     src = O7;
1402   } else {
1403     disp_reg = addr->index()->as_pointer_register();
1404     assert(disp_value == 0, "can't handle 3 operand addresses");
1405   }
1406 
1407   // remember the offset of the store.  The patching_epilog must be done
1408   // before the call to add_debug_info_for_null_check, otherwise the PcDescs don't get
1409   // entered in increasing order.
1410   int offset;
1411 
1412   assert(disp_reg != noreg || Assembler::is_simm13(disp_value), "should have set this up");
1413   if (disp_reg == noreg) {
1414     offset = store(from_reg, src, disp_value, type, wide, unaligned);
1415   } else {
1416     assert(!unaligned, "can't handle this");
1417     offset = store(from_reg, src, disp_reg, type, wide);
1418   }
1419 
1420   if (patch != NULL) {
1421     patching_epilog(patch, patch_code, src, info);
1422   }
1423 
1424   if (info != NULL) add_debug_info_for_null_check(offset, info);
1425 }
1426 
1427 
1428 void LIR_Assembler::return_op(LIR_Opr result) {
1429   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
1430     __ reserved_stack_check();
1431   }
1432   if (SafepointMechanism::uses_thread_local_poll()) {
1433     __ ld_ptr(Address(G2_thread, Thread::polling_page_offset()), L0);
1434   } else {
1435     __ set((intptr_t)os::get_polling_page(), L0);
1436   }
1437   __ relocate(relocInfo::poll_return_type);
1438   __ ld_ptr(L0, 0, G0);
1439   __ ret();
1440   __ delayed()->restore();
1441 }
1442 
1443 
1444 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
1445   if (SafepointMechanism::uses_thread_local_poll()) {
1446     __ ld_ptr(Address(G2_thread, Thread::polling_page_offset()), tmp->as_register());
1447   } else {
1448     __ set((intptr_t)os::get_polling_page(), tmp->as_register());
1449   }
1450   if (info != NULL) {
1451     add_debug_info_for_branch(info);
1452   }
1453   int offset = __ offset();
1454 
1455   __ relocate(relocInfo::poll_type);
1456   __ ld_ptr(tmp->as_register(), 0, G0);
1457   return offset;
1458 }
1459 
1460 
1461 void LIR_Assembler::emit_static_call_stub() {
1462   address call_pc = __ pc();
1463   address stub = __ start_a_stub(call_stub_size());
1464   if (stub == NULL) {
1465     bailout("static call stub overflow");
1466     return;
1467   }
1468 
1469   int start = __ offset();
1470   __ relocate(static_stub_Relocation::spec(call_pc));
1471 
1472   __ set_metadata(NULL, G5);
1473   // must be set to -1 at code generation time
1474   AddressLiteral addrlit(-1);
1475   __ jump_to(addrlit, G3);
1476   __ delayed()->nop();
1477 
1478   assert(__ offset() - start <= call_stub_size(), "stub too big");
1479   __ end_a_stub();
1480 }
1481 
1482 
1483 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1484   if (opr1->is_single_fpu()) {
1485     __ fcmp(FloatRegisterImpl::S, Assembler::fcc0, opr1->as_float_reg(), opr2->as_float_reg());
1486   } else if (opr1->is_double_fpu()) {
1487     __ fcmp(FloatRegisterImpl::D, Assembler::fcc0, opr1->as_double_reg(), opr2->as_double_reg());
1488   } else if (opr1->is_single_cpu()) {
1489     if (opr2->is_constant()) {
1490       switch (opr2->as_constant_ptr()->type()) {
1491         case T_INT:
1492           { jint con = opr2->as_constant_ptr()->as_jint();
1493             if (Assembler::is_simm13(con)) {
1494               __ cmp(opr1->as_register(), con);
1495             } else {
1496               __ set(con, O7);
1497               __ cmp(opr1->as_register(), O7);
1498             }
1499           }
1500           break;
1501 
1502         case T_OBJECT:
1503           // there are only equal/notequal comparisions on objects
1504           { jobject con = opr2->as_constant_ptr()->as_jobject();
1505             if (con == NULL) {
1506               __ cmp(opr1->as_register(), 0);
1507             } else {
1508               jobject2reg(con, O7);
1509               __ cmp(opr1->as_register(), O7);
1510             }
1511           }
1512           break;
1513 
1514         default:
1515           ShouldNotReachHere();
1516           break;
1517       }
1518     } else {
1519       if (opr2->is_address()) {
1520         LIR_Address * addr = opr2->as_address_ptr();
1521         BasicType type = addr->type();
1522         if ( type == T_OBJECT ) __ ld_ptr(as_Address(addr), O7);
1523         else                    __ ld(as_Address(addr), O7);
1524         __ cmp(opr1->as_register(), O7);
1525       } else {
1526         __ cmp(opr1->as_register(), opr2->as_register());
1527       }
1528     }
1529   } else if (opr1->is_double_cpu()) {
1530     Register xlo = opr1->as_register_lo();
1531     Register xhi = opr1->as_register_hi();
1532     if (opr2->is_constant() && opr2->as_jlong() == 0) {
1533       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles these cases");
1534       __ orcc(xhi, G0, G0);
1535     } else if (opr2->is_register()) {
1536       Register ylo = opr2->as_register_lo();
1537       Register yhi = opr2->as_register_hi();
1538       __ cmp(xlo, ylo);
1539     } else {
1540       ShouldNotReachHere();
1541     }
1542   } else if (opr1->is_address()) {
1543     LIR_Address * addr = opr1->as_address_ptr();
1544     BasicType type = addr->type();
1545     assert (opr2->is_constant(), "Checking");
1546     if ( type == T_OBJECT ) __ ld_ptr(as_Address(addr), O7);
1547     else                    __ ld(as_Address(addr), O7);
1548     __ cmp(O7, opr2->as_constant_ptr()->as_jint());
1549   } else {
1550     ShouldNotReachHere();
1551   }
1552 }
1553 
1554 
1555 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1556   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1557     bool is_unordered_less = (code == lir_ucmp_fd2i);
1558     if (left->is_single_fpu()) {
1559       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1560     } else if (left->is_double_fpu()) {
1561       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1562     } else {
1563       ShouldNotReachHere();
1564     }
1565   } else if (code == lir_cmp_l2i) {
1566     __ lcmp(left->as_register_lo(), right->as_register_lo(), dst->as_register());
1567   } else {
1568     ShouldNotReachHere();
1569   }
1570 }
1571 
1572 
1573 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1574   Assembler::Condition acond;
1575   switch (condition) {
1576     case lir_cond_equal:        acond = Assembler::equal;        break;
1577     case lir_cond_notEqual:     acond = Assembler::notEqual;     break;
1578     case lir_cond_less:         acond = Assembler::less;         break;
1579     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    break;
1580     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; break;
1581     case lir_cond_greater:      acond = Assembler::greater;      break;
1582     case lir_cond_aboveEqual:   acond = Assembler::greaterEqualUnsigned;      break;
1583     case lir_cond_belowEqual:   acond = Assembler::lessEqualUnsigned;      break;
1584     default:                         ShouldNotReachHere();
1585   };
1586 
1587   if (opr1->is_constant() && opr1->type() == T_INT) {
1588     Register dest = result->as_register();
1589     // load up first part of constant before branch
1590     // and do the rest in the delay slot.
1591     if (!Assembler::is_simm13(opr1->as_jint())) {
1592       __ sethi(opr1->as_jint(), dest);
1593     }
1594   } else if (opr1->is_constant()) {
1595     const2reg(opr1, result, lir_patch_none, NULL);
1596   } else if (opr1->is_register()) {
1597     reg2reg(opr1, result);
1598   } else if (opr1->is_stack()) {
1599     stack2reg(opr1, result, result->type());
1600   } else {
1601     ShouldNotReachHere();
1602   }
1603   Label skip;
1604     if  (type == T_INT) {
1605       __ br(acond, false, Assembler::pt, skip);
1606     } else {
1607       __ brx(acond, false, Assembler::pt, skip); // checks icc on 32bit and xcc on 64bit
1608     }
1609   if (opr1->is_constant() && opr1->type() == T_INT) {
1610     Register dest = result->as_register();
1611     if (Assembler::is_simm13(opr1->as_jint())) {
1612       __ delayed()->or3(G0, opr1->as_jint(), dest);
1613     } else {
1614       // the sethi has been done above, so just put in the low 10 bits
1615       __ delayed()->or3(dest, opr1->as_jint() & 0x3ff, dest);
1616     }
1617   } else {
1618     // can't do anything useful in the delay slot
1619     __ delayed()->nop();
1620   }
1621   if (opr2->is_constant()) {
1622     const2reg(opr2, result, lir_patch_none, NULL);
1623   } else if (opr2->is_register()) {
1624     reg2reg(opr2, result);
1625   } else if (opr2->is_stack()) {
1626     stack2reg(opr2, result, result->type());
1627   } else {
1628     ShouldNotReachHere();
1629   }
1630   __ bind(skip);
1631 }
1632 
1633 
1634 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1635   assert(info == NULL, "unused on this code path");
1636   assert(left->is_register(), "wrong items state");
1637   assert(dest->is_register(), "wrong items state");
1638 
1639   if (right->is_register()) {
1640     if (dest->is_float_kind()) {
1641 
1642       FloatRegister lreg, rreg, res;
1643       FloatRegisterImpl::Width w;
1644       if (right->is_single_fpu()) {
1645         w = FloatRegisterImpl::S;
1646         lreg = left->as_float_reg();
1647         rreg = right->as_float_reg();
1648         res  = dest->as_float_reg();
1649       } else {
1650         w = FloatRegisterImpl::D;
1651         lreg = left->as_double_reg();
1652         rreg = right->as_double_reg();
1653         res  = dest->as_double_reg();
1654       }
1655 
1656       switch (code) {
1657         case lir_add: __ fadd(w, lreg, rreg, res); break;
1658         case lir_sub: __ fsub(w, lreg, rreg, res); break;
1659         case lir_mul: // fall through
1660         case lir_mul_strictfp: __ fmul(w, lreg, rreg, res); break;
1661         case lir_div: // fall through
1662         case lir_div_strictfp: __ fdiv(w, lreg, rreg, res); break;
1663         default: ShouldNotReachHere();
1664       }
1665 
1666     } else if (dest->is_double_cpu()) {
1667       Register dst_lo = dest->as_register_lo();
1668       Register op1_lo = left->as_pointer_register();
1669       Register op2_lo = right->as_pointer_register();
1670 
1671       switch (code) {
1672         case lir_add:
1673           __ add(op1_lo, op2_lo, dst_lo);
1674           break;
1675 
1676         case lir_sub:
1677           __ sub(op1_lo, op2_lo, dst_lo);
1678           break;
1679 
1680         default: ShouldNotReachHere();
1681       }
1682     } else {
1683       assert (right->is_single_cpu(), "Just Checking");
1684 
1685       Register lreg = left->as_register();
1686       Register res  = dest->as_register();
1687       Register rreg = right->as_register();
1688       switch (code) {
1689         case lir_add:  __ add  (lreg, rreg, res); break;
1690         case lir_sub:  __ sub  (lreg, rreg, res); break;
1691         case lir_mul:  __ mulx (lreg, rreg, res); break;
1692         default: ShouldNotReachHere();
1693       }
1694     }
1695   } else {
1696     assert (right->is_constant(), "must be constant");
1697 
1698     if (dest->is_single_cpu()) {
1699       Register lreg = left->as_register();
1700       Register res  = dest->as_register();
1701       int    simm13 = right->as_constant_ptr()->as_jint();
1702 
1703       switch (code) {
1704         case lir_add:  __ add  (lreg, simm13, res); break;
1705         case lir_sub:  __ sub  (lreg, simm13, res); break;
1706         case lir_mul:  __ mulx (lreg, simm13, res); break;
1707         default: ShouldNotReachHere();
1708       }
1709     } else {
1710       Register lreg = left->as_pointer_register();
1711       Register res  = dest->as_register_lo();
1712       long con = right->as_constant_ptr()->as_jlong();
1713       assert(Assembler::is_simm13(con), "must be simm13");
1714 
1715       switch (code) {
1716         case lir_add:  __ add  (lreg, (int)con, res); break;
1717         case lir_sub:  __ sub  (lreg, (int)con, res); break;
1718         case lir_mul:  __ mulx (lreg, (int)con, res); break;
1719         default: ShouldNotReachHere();
1720       }
1721     }
1722   }
1723 }
1724 
1725 
1726 void LIR_Assembler::fpop() {
1727   // do nothing
1728 }
1729 
1730 
1731 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr thread, LIR_Opr dest, LIR_Op* op) {
1732   switch (code) {
1733     case lir_tan: {
1734       assert(thread->is_valid(), "preserve the thread object for performance reasons");
1735       assert(dest->as_double_reg() == F0, "the result will be in f0/f1");
1736       break;
1737     }
1738     case lir_sqrt: {
1739       assert(!thread->is_valid(), "there is no need for a thread_reg for dsqrt");
1740       FloatRegister src_reg = value->as_double_reg();
1741       FloatRegister dst_reg = dest->as_double_reg();
1742       __ fsqrt(FloatRegisterImpl::D, src_reg, dst_reg);
1743       break;
1744     }
1745     case lir_abs: {
1746       assert(!thread->is_valid(), "there is no need for a thread_reg for fabs");
1747       FloatRegister src_reg = value->as_double_reg();
1748       FloatRegister dst_reg = dest->as_double_reg();
1749       __ fabs(FloatRegisterImpl::D, src_reg, dst_reg);
1750       break;
1751     }
1752     default: {
1753       ShouldNotReachHere();
1754       break;
1755     }
1756   }
1757 }
1758 
1759 
1760 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) {
1761   if (right->is_constant()) {
1762     if (dest->is_single_cpu()) {
1763       int simm13 = right->as_constant_ptr()->as_jint();
1764       switch (code) {
1765         case lir_logic_and:   __ and3 (left->as_register(), simm13, dest->as_register()); break;
1766         case lir_logic_or:    __ or3  (left->as_register(), simm13, dest->as_register()); break;
1767         case lir_logic_xor:   __ xor3 (left->as_register(), simm13, dest->as_register()); break;
1768         default: ShouldNotReachHere();
1769       }
1770     } else {
1771       long c = right->as_constant_ptr()->as_jlong();
1772       assert(c == (int)c && Assembler::is_simm13(c), "out of range");
1773       int simm13 = (int)c;
1774       switch (code) {
1775         case lir_logic_and:
1776           __ and3 (left->as_register_lo(), simm13, dest->as_register_lo());
1777           break;
1778 
1779         case lir_logic_or:
1780           __ or3 (left->as_register_lo(), simm13, dest->as_register_lo());
1781           break;
1782 
1783         case lir_logic_xor:
1784           __ xor3 (left->as_register_lo(), simm13, dest->as_register_lo());
1785           break;
1786 
1787         default: ShouldNotReachHere();
1788       }
1789     }
1790   } else {
1791     assert(right->is_register(), "right should be in register");
1792 
1793     if (dest->is_single_cpu()) {
1794       switch (code) {
1795         case lir_logic_and:   __ and3 (left->as_register(), right->as_register(), dest->as_register()); break;
1796         case lir_logic_or:    __ or3  (left->as_register(), right->as_register(), dest->as_register()); break;
1797         case lir_logic_xor:   __ xor3 (left->as_register(), right->as_register(), dest->as_register()); break;
1798         default: ShouldNotReachHere();
1799       }
1800     } else {
1801       Register l = (left->is_single_cpu() && left->is_oop_register()) ? left->as_register() :
1802                                                                         left->as_register_lo();
1803       Register r = (right->is_single_cpu() && right->is_oop_register()) ? right->as_register() :
1804                                                                           right->as_register_lo();
1805 
1806       switch (code) {
1807         case lir_logic_and: __ and3 (l, r, dest->as_register_lo()); break;
1808         case lir_logic_or:  __ or3  (l, r, dest->as_register_lo()); break;
1809         case lir_logic_xor: __ xor3 (l, r, dest->as_register_lo()); break;
1810         default: ShouldNotReachHere();
1811       }
1812     }
1813   }
1814 }
1815 
1816 
1817 int LIR_Assembler::shift_amount(BasicType t) {
1818   int elem_size = type2aelembytes(t);
1819   switch (elem_size) {
1820     case 1 : return 0;
1821     case 2 : return 1;
1822     case 4 : return 2;
1823     case 8 : return 3;
1824   }
1825   ShouldNotReachHere();
1826   return -1;
1827 }
1828 
1829 
1830 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
1831   assert(exceptionOop->as_register() == Oexception, "should match");
1832   assert(exceptionPC->as_register() == Oissuing_pc, "should match");
1833 
1834   info->add_register_oop(exceptionOop);
1835 
1836   // reuse the debug info from the safepoint poll for the throw op itself
1837   address pc_for_athrow  = __ pc();
1838   int pc_for_athrow_offset = __ offset();
1839   RelocationHolder rspec = internal_word_Relocation::spec(pc_for_athrow);
1840   __ set(pc_for_athrow, Oissuing_pc, rspec);
1841   add_call_info(pc_for_athrow_offset, info); // for exception handler
1842 
1843   __ call(Runtime1::entry_for(Runtime1::handle_exception_id), relocInfo::runtime_call_type);
1844   __ delayed()->nop();
1845 }
1846 
1847 
1848 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
1849   assert(exceptionOop->as_register() == Oexception, "should match");
1850 
1851   __ br(Assembler::always, false, Assembler::pt, _unwind_handler_entry);
1852   __ delayed()->nop();
1853 }
1854 
1855 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
1856   Register src = op->src()->as_register();
1857   Register dst = op->dst()->as_register();
1858   Register src_pos = op->src_pos()->as_register();
1859   Register dst_pos = op->dst_pos()->as_register();
1860   Register length  = op->length()->as_register();
1861   Register tmp = op->tmp()->as_register();
1862   Register tmp2 = O7;
1863 
1864   int flags = op->flags();
1865   ciArrayKlass* default_type = op->expected_type();
1866   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
1867   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
1868 
1869   // higher 32bits must be null
1870   __ sra(dst_pos, 0, dst_pos);
1871   __ sra(src_pos, 0, src_pos);
1872   __ sra(length, 0, length);
1873 
1874   // set up the arraycopy stub information
1875   ArrayCopyStub* stub = op->stub();
1876 
1877   // always do stub if no type information is available.  it's ok if
1878   // the known type isn't loaded since the code sanity checks
1879   // in debug mode and the type isn't required when we know the exact type
1880   // also check that the type is an array type.
1881   if (op->expected_type() == NULL) {
1882     __ mov(src,     O0);
1883     __ mov(src_pos, O1);
1884     __ mov(dst,     O2);
1885     __ mov(dst_pos, O3);
1886     __ mov(length,  O4);
1887     address copyfunc_addr = StubRoutines::generic_arraycopy();
1888     assert(copyfunc_addr != NULL, "generic arraycopy stub required");
1889 
1890 #ifndef PRODUCT
1891     if (PrintC1Statistics) {
1892       address counter = (address)&Runtime1::_generic_arraycopystub_cnt;
1893       __ inc_counter(counter, G1, G3);
1894     }
1895 #endif
1896     __ call_VM_leaf(tmp, copyfunc_addr);
1897 
1898     __ xor3(O0, -1, tmp);
1899     __ sub(length, tmp, length);
1900     __ add(src_pos, tmp, src_pos);
1901     __ cmp_zero_and_br(Assembler::less, O0, *stub->entry());
1902     __ delayed()->add(dst_pos, tmp, dst_pos);
1903     __ bind(*stub->continuation());
1904     return;
1905   }
1906 
1907   assert(default_type != NULL && default_type->is_array_klass(), "must be true at this point");
1908 
1909   // make sure src and dst are non-null and load array length
1910   if (flags & LIR_OpArrayCopy::src_null_check) {
1911     __ tst(src);
1912     __ brx(Assembler::equal, false, Assembler::pn, *stub->entry());
1913     __ delayed()->nop();
1914   }
1915 
1916   if (flags & LIR_OpArrayCopy::dst_null_check) {
1917     __ tst(dst);
1918     __ brx(Assembler::equal, false, Assembler::pn, *stub->entry());
1919     __ delayed()->nop();
1920   }
1921 
1922   // If the compiler was not able to prove that exact type of the source or the destination
1923   // of the arraycopy is an array type, check at runtime if the source or the destination is
1924   // an instance type.
1925   if (flags & LIR_OpArrayCopy::type_check) {
1926     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
1927       __ load_klass(dst, tmp);
1928       __ lduw(tmp, in_bytes(Klass::layout_helper_offset()), tmp2);
1929       __ cmp(tmp2, Klass::_lh_neutral_value);
1930       __ br(Assembler::greaterEqual, false, Assembler::pn, *stub->entry());
1931       __ delayed()->nop();
1932     }
1933 
1934     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
1935       __ load_klass(src, tmp);
1936       __ lduw(tmp, in_bytes(Klass::layout_helper_offset()), tmp2);
1937       __ cmp(tmp2, Klass::_lh_neutral_value);
1938       __ br(Assembler::greaterEqual, false, Assembler::pn, *stub->entry());
1939       __ delayed()->nop();
1940     }
1941   }
1942 
1943   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
1944     // test src_pos register
1945     __ cmp_zero_and_br(Assembler::less, src_pos, *stub->entry());
1946     __ delayed()->nop();
1947   }
1948 
1949   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
1950     // test dst_pos register
1951     __ cmp_zero_and_br(Assembler::less, dst_pos, *stub->entry());
1952     __ delayed()->nop();
1953   }
1954 
1955   if (flags & LIR_OpArrayCopy::length_positive_check) {
1956     // make sure length isn't negative
1957     __ cmp_zero_and_br(Assembler::less, length, *stub->entry());
1958     __ delayed()->nop();
1959   }
1960 
1961   if (flags & LIR_OpArrayCopy::src_range_check) {
1962     __ ld(src, arrayOopDesc::length_offset_in_bytes(), tmp2);
1963     __ add(length, src_pos, tmp);
1964     __ cmp(tmp2, tmp);
1965     __ br(Assembler::carrySet, false, Assembler::pn, *stub->entry());
1966     __ delayed()->nop();
1967   }
1968 
1969   if (flags & LIR_OpArrayCopy::dst_range_check) {
1970     __ ld(dst, arrayOopDesc::length_offset_in_bytes(), tmp2);
1971     __ add(length, dst_pos, tmp);
1972     __ cmp(tmp2, tmp);
1973     __ br(Assembler::carrySet, false, Assembler::pn, *stub->entry());
1974     __ delayed()->nop();
1975   }
1976 
1977   int shift = shift_amount(basic_type);
1978 
1979   if (flags & LIR_OpArrayCopy::type_check) {
1980     // We don't know the array types are compatible
1981     if (basic_type != T_OBJECT) {
1982       // Simple test for basic type arrays
1983       if (UseCompressedClassPointers) {
1984         // We don't need decode because we just need to compare
1985         __ lduw(src, oopDesc::klass_offset_in_bytes(), tmp);
1986         __ lduw(dst, oopDesc::klass_offset_in_bytes(), tmp2);
1987         __ cmp(tmp, tmp2);
1988         __ br(Assembler::notEqual, false, Assembler::pt, *stub->entry());
1989       } else {
1990         __ ld_ptr(src, oopDesc::klass_offset_in_bytes(), tmp);
1991         __ ld_ptr(dst, oopDesc::klass_offset_in_bytes(), tmp2);
1992         __ cmp(tmp, tmp2);
1993         __ brx(Assembler::notEqual, false, Assembler::pt, *stub->entry());
1994       }
1995       __ delayed()->nop();
1996     } else {
1997       // For object arrays, if src is a sub class of dst then we can
1998       // safely do the copy.
1999       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2000 
2001       Label cont, slow;
2002       assert_different_registers(tmp, tmp2, G3, G1);
2003 
2004       __ load_klass(src, G3);
2005       __ load_klass(dst, G1);
2006 
2007       __ check_klass_subtype_fast_path(G3, G1, tmp, tmp2, &cont, copyfunc_addr == NULL ? stub->entry() : &slow, NULL);
2008 
2009       __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
2010       __ delayed()->nop();
2011 
2012       __ cmp(G3, 0);
2013       if (copyfunc_addr != NULL) { // use stub if available
2014         // src is not a sub class of dst so we have to do a
2015         // per-element check.
2016         __ br(Assembler::notEqual, false, Assembler::pt, cont);
2017         __ delayed()->nop();
2018 
2019         __ bind(slow);
2020 
2021         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2022         if ((flags & mask) != mask) {
2023           // Check that at least both of them object arrays.
2024           assert(flags & mask, "one of the two should be known to be an object array");
2025 
2026           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2027             __ load_klass(src, tmp);
2028           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2029             __ load_klass(dst, tmp);
2030           }
2031           int lh_offset = in_bytes(Klass::layout_helper_offset());
2032 
2033           __ lduw(tmp, lh_offset, tmp2);
2034 
2035           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2036           __ set(objArray_lh, tmp);
2037           __ cmp(tmp, tmp2);
2038           __ br(Assembler::notEqual, false, Assembler::pt,  *stub->entry());
2039           __ delayed()->nop();
2040         }
2041 
2042         Register src_ptr = O0;
2043         Register dst_ptr = O1;
2044         Register len     = O2;
2045         Register chk_off = O3;
2046         Register super_k = O4;
2047 
2048         __ add(src, arrayOopDesc::base_offset_in_bytes(basic_type), src_ptr);
2049         if (shift == 0) {
2050           __ add(src_ptr, src_pos, src_ptr);
2051         } else {
2052           __ sll(src_pos, shift, tmp);
2053           __ add(src_ptr, tmp, src_ptr);
2054         }
2055 
2056         __ add(dst, arrayOopDesc::base_offset_in_bytes(basic_type), dst_ptr);
2057         if (shift == 0) {
2058           __ add(dst_ptr, dst_pos, dst_ptr);
2059         } else {
2060           __ sll(dst_pos, shift, tmp);
2061           __ add(dst_ptr, tmp, dst_ptr);
2062         }
2063         __ mov(length, len);
2064         __ load_klass(dst, tmp);
2065 
2066         int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2067         __ ld_ptr(tmp, ek_offset, super_k);
2068 
2069         int sco_offset = in_bytes(Klass::super_check_offset_offset());
2070         __ lduw(super_k, sco_offset, chk_off);
2071 
2072         __ call_VM_leaf(tmp, copyfunc_addr);
2073 
2074 #ifndef PRODUCT
2075         if (PrintC1Statistics) {
2076           Label failed;
2077           __ br_notnull_short(O0, Assembler::pn, failed);
2078           __ inc_counter((address)&Runtime1::_arraycopy_checkcast_cnt, G1, G3);
2079           __ bind(failed);
2080         }
2081 #endif
2082 
2083         __ br_null(O0, false, Assembler::pt,  *stub->continuation());
2084         __ delayed()->xor3(O0, -1, tmp);
2085 
2086 #ifndef PRODUCT
2087         if (PrintC1Statistics) {
2088           __ inc_counter((address)&Runtime1::_arraycopy_checkcast_attempt_cnt, G1, G3);
2089         }
2090 #endif
2091 
2092         __ sub(length, tmp, length);
2093         __ add(src_pos, tmp, src_pos);
2094         __ br(Assembler::always, false, Assembler::pt, *stub->entry());
2095         __ delayed()->add(dst_pos, tmp, dst_pos);
2096 
2097         __ bind(cont);
2098       } else {
2099         __ br(Assembler::equal, false, Assembler::pn, *stub->entry());
2100         __ delayed()->nop();
2101         __ bind(cont);
2102       }
2103     }
2104   }
2105 
2106 #ifdef ASSERT
2107   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2108     // Sanity check the known type with the incoming class.  For the
2109     // primitive case the types must match exactly with src.klass and
2110     // dst.klass each exactly matching the default type.  For the
2111     // object array case, if no type check is needed then either the
2112     // dst type is exactly the expected type and the src type is a
2113     // subtype which we can't check or src is the same array as dst
2114     // but not necessarily exactly of type default_type.
2115     Label known_ok, halt;
2116     metadata2reg(op->expected_type()->constant_encoding(), tmp);
2117     if (UseCompressedClassPointers) {
2118       // tmp holds the default type. It currently comes uncompressed after the
2119       // load of a constant, so encode it.
2120       __ encode_klass_not_null(tmp);
2121       // load the raw value of the dst klass, since we will be comparing
2122       // uncompressed values directly.
2123       __ lduw(dst, oopDesc::klass_offset_in_bytes(), tmp2);
2124       if (basic_type != T_OBJECT) {
2125         __ cmp(tmp, tmp2);
2126         __ br(Assembler::notEqual, false, Assembler::pn, halt);
2127         // load the raw value of the src klass.
2128         __ delayed()->lduw(src, oopDesc::klass_offset_in_bytes(), tmp2);
2129         __ cmp_and_br_short(tmp, tmp2, Assembler::equal, Assembler::pn, known_ok);
2130       } else {
2131         __ cmp(tmp, tmp2);
2132         __ br(Assembler::equal, false, Assembler::pn, known_ok);
2133         __ delayed()->cmp(src, dst);
2134         __ brx(Assembler::equal, false, Assembler::pn, known_ok);
2135         __ delayed()->nop();
2136       }
2137     } else {
2138       __ ld_ptr(dst, oopDesc::klass_offset_in_bytes(), tmp2);
2139       if (basic_type != T_OBJECT) {
2140         __ cmp(tmp, tmp2);
2141         __ brx(Assembler::notEqual, false, Assembler::pn, halt);
2142         __ delayed()->ld_ptr(src, oopDesc::klass_offset_in_bytes(), tmp2);
2143         __ cmp_and_brx_short(tmp, tmp2, Assembler::equal, Assembler::pn, known_ok);
2144       } else {
2145         __ cmp(tmp, tmp2);
2146         __ brx(Assembler::equal, false, Assembler::pn, known_ok);
2147         __ delayed()->cmp(src, dst);
2148         __ brx(Assembler::equal, false, Assembler::pn, known_ok);
2149         __ delayed()->nop();
2150       }
2151     }
2152     __ bind(halt);
2153     __ stop("incorrect type information in arraycopy");
2154     __ bind(known_ok);
2155   }
2156 #endif
2157 
2158 #ifndef PRODUCT
2159   if (PrintC1Statistics) {
2160     address counter = Runtime1::arraycopy_count_address(basic_type);
2161     __ inc_counter(counter, G1, G3);
2162   }
2163 #endif
2164 
2165   Register src_ptr = O0;
2166   Register dst_ptr = O1;
2167   Register len     = O2;
2168 
2169   __ add(src, arrayOopDesc::base_offset_in_bytes(basic_type), src_ptr);
2170   if (shift == 0) {
2171     __ add(src_ptr, src_pos, src_ptr);
2172   } else {
2173     __ sll(src_pos, shift, tmp);
2174     __ add(src_ptr, tmp, src_ptr);
2175   }
2176 
2177   __ add(dst, arrayOopDesc::base_offset_in_bytes(basic_type), dst_ptr);
2178   if (shift == 0) {
2179     __ add(dst_ptr, dst_pos, dst_ptr);
2180   } else {
2181     __ sll(dst_pos, shift, tmp);
2182     __ add(dst_ptr, tmp, dst_ptr);
2183   }
2184 
2185   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2186   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2187   const char *name;
2188   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2189 
2190   // arraycopy stubs takes a length in number of elements, so don't scale it.
2191   __ mov(length, len);
2192   __ call_VM_leaf(tmp, entry);
2193 
2194   __ bind(*stub->continuation());
2195 }
2196 
2197 
2198 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2199   if (dest->is_single_cpu()) {
2200     if (left->type() == T_OBJECT) {
2201       switch (code) {
2202         case lir_shl:  __ sllx  (left->as_register(), count->as_register(), dest->as_register()); break;
2203         case lir_shr:  __ srax  (left->as_register(), count->as_register(), dest->as_register()); break;
2204         case lir_ushr: __ srl   (left->as_register(), count->as_register(), dest->as_register()); break;
2205         default: ShouldNotReachHere();
2206       }
2207     } else
2208       switch (code) {
2209         case lir_shl:  __ sll   (left->as_register(), count->as_register(), dest->as_register()); break;
2210         case lir_shr:  __ sra   (left->as_register(), count->as_register(), dest->as_register()); break;
2211         case lir_ushr: __ srl   (left->as_register(), count->as_register(), dest->as_register()); break;
2212         default: ShouldNotReachHere();
2213       }
2214   } else {
2215     switch (code) {
2216       case lir_shl:  __ sllx  (left->as_register_lo(), count->as_register(), dest->as_register_lo()); break;
2217       case lir_shr:  __ srax  (left->as_register_lo(), count->as_register(), dest->as_register_lo()); break;
2218       case lir_ushr: __ srlx  (left->as_register_lo(), count->as_register(), dest->as_register_lo()); break;
2219       default: ShouldNotReachHere();
2220     }
2221   }
2222 }
2223 
2224 
2225 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2226   if (left->type() == T_OBJECT) {
2227     count = count & 63;  // shouldn't shift by more than sizeof(intptr_t)
2228     Register l = left->as_register();
2229     Register d = dest->as_register_lo();
2230     switch (code) {
2231       case lir_shl:  __ sllx  (l, count, d); break;
2232       case lir_shr:  __ srax  (l, count, d); break;
2233       case lir_ushr: __ srlx  (l, count, d); break;
2234       default: ShouldNotReachHere();
2235     }
2236     return;
2237   }
2238 
2239   if (dest->is_single_cpu()) {
2240     count = count & 0x1F; // Java spec
2241     switch (code) {
2242       case lir_shl:  __ sll   (left->as_register(), count, dest->as_register()); break;
2243       case lir_shr:  __ sra   (left->as_register(), count, dest->as_register()); break;
2244       case lir_ushr: __ srl   (left->as_register(), count, dest->as_register()); break;
2245       default: ShouldNotReachHere();
2246     }
2247   } else if (dest->is_double_cpu()) {
2248     count = count & 63; // Java spec
2249     switch (code) {
2250       case lir_shl:  __ sllx  (left->as_pointer_register(), count, dest->as_pointer_register()); break;
2251       case lir_shr:  __ srax  (left->as_pointer_register(), count, dest->as_pointer_register()); break;
2252       case lir_ushr: __ srlx  (left->as_pointer_register(), count, dest->as_pointer_register()); break;
2253       default: ShouldNotReachHere();
2254     }
2255   } else {
2256     ShouldNotReachHere();
2257   }
2258 }
2259 
2260 
2261 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
2262   assert(op->tmp1()->as_register()  == G1 &&
2263          op->tmp2()->as_register()  == G3 &&
2264          op->tmp3()->as_register()  == G4 &&
2265          op->obj()->as_register()   == O0 &&
2266          op->klass()->as_register() == G5, "must be");
2267   if (op->init_check()) {
2268     add_debug_info_for_null_check_here(op->stub()->info());
2269     __ ldub(op->klass()->as_register(),
2270           in_bytes(InstanceKlass::init_state_offset()),
2271           op->tmp1()->as_register());
2272     __ cmp(op->tmp1()->as_register(), InstanceKlass::fully_initialized);
2273     __ br(Assembler::notEqual, false, Assembler::pn, *op->stub()->entry());
2274     __ delayed()->nop();
2275   }
2276   __ allocate_object(op->obj()->as_register(),
2277                      op->tmp1()->as_register(),
2278                      op->tmp2()->as_register(),
2279                      op->tmp3()->as_register(),
2280                      op->header_size(),
2281                      op->object_size(),
2282                      op->klass()->as_register(),
2283                      *op->stub()->entry());
2284   __ bind(*op->stub()->continuation());
2285   __ verify_oop(op->obj()->as_register());
2286 }
2287 
2288 
2289 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
2290   assert(op->tmp1()->as_register()  == G1 &&
2291          op->tmp2()->as_register()  == G3 &&
2292          op->tmp3()->as_register()  == G4 &&
2293          op->tmp4()->as_register()  == O1 &&
2294          op->klass()->as_register() == G5, "must be");
2295 
2296   __ signx(op->len()->as_register());
2297   if (UseSlowPath ||
2298       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
2299       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
2300     __ br(Assembler::always, false, Assembler::pt, *op->stub()->entry());
2301     __ delayed()->nop();
2302   } else {
2303     __ allocate_array(op->obj()->as_register(),
2304                       op->len()->as_register(),
2305                       op->tmp1()->as_register(),
2306                       op->tmp2()->as_register(),
2307                       op->tmp3()->as_register(),
2308                       arrayOopDesc::header_size(op->type()),
2309                       type2aelembytes(op->type()),
2310                       op->klass()->as_register(),
2311                       *op->stub()->entry());
2312   }
2313   __ bind(*op->stub()->continuation());
2314 }
2315 
2316 
2317 void LIR_Assembler::type_profile_helper(Register mdo, int mdo_offset_bias,
2318                                         ciMethodData *md, ciProfileData *data,
2319                                         Register recv, Register tmp1, Label* update_done) {
2320   uint i;
2321   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2322     Label next_test;
2323     // See if the receiver is receiver[n].
2324     Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
2325                           mdo_offset_bias);
2326     __ ld_ptr(receiver_addr, tmp1);
2327     __ verify_klass_ptr(tmp1);
2328     __ cmp_and_brx_short(recv, tmp1, Assembler::notEqual, Assembler::pt, next_test);
2329     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
2330                       mdo_offset_bias);
2331     __ ld_ptr(data_addr, tmp1);
2332     __ add(tmp1, DataLayout::counter_increment, tmp1);
2333     __ st_ptr(tmp1, data_addr);
2334     __ ba(*update_done);
2335     __ delayed()->nop();
2336     __ bind(next_test);
2337   }
2338 
2339   // Didn't find receiver; find next empty slot and fill it in
2340   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2341     Label next_test;
2342     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
2343                       mdo_offset_bias);
2344     __ ld_ptr(recv_addr, tmp1);
2345     __ br_notnull_short(tmp1, Assembler::pt, next_test);
2346     __ st_ptr(recv, recv_addr);
2347     __ set(DataLayout::counter_increment, tmp1);
2348     __ st_ptr(tmp1, mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
2349               mdo_offset_bias);
2350     __ ba(*update_done);
2351     __ delayed()->nop();
2352     __ bind(next_test);
2353   }
2354 }
2355 
2356 
2357 void LIR_Assembler::setup_md_access(ciMethod* method, int bci,
2358                                     ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) {
2359   md = method->method_data_or_null();
2360   assert(md != NULL, "Sanity");
2361   data = md->bci_to_data(bci);
2362   assert(data != NULL,       "need data for checkcast");
2363   assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
2364   if (!Assembler::is_simm13(md->byte_offset_of_slot(data, DataLayout::header_offset()) + data->size_in_bytes())) {
2365     // The offset is large so bias the mdo by the base of the slot so
2366     // that the ld can use simm13s to reference the slots of the data
2367     mdo_offset_bias = md->byte_offset_of_slot(data, DataLayout::header_offset());
2368   }
2369 }
2370 
2371 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
2372   // we always need a stub for the failure case.
2373   CodeStub* stub = op->stub();
2374   Register obj = op->object()->as_register();
2375   Register k_RInfo = op->tmp1()->as_register();
2376   Register klass_RInfo = op->tmp2()->as_register();
2377   Register dst = op->result_opr()->as_register();
2378   Register Rtmp1 = op->tmp3()->as_register();
2379   ciKlass* k = op->klass();
2380 
2381 
2382   if (obj == k_RInfo) {
2383     k_RInfo = klass_RInfo;
2384     klass_RInfo = obj;
2385   }
2386 
2387   ciMethodData* md;
2388   ciProfileData* data;
2389   int mdo_offset_bias = 0;
2390   if (op->should_profile()) {
2391     ciMethod* method = op->profiled_method();
2392     assert(method != NULL, "Should have method");
2393     setup_md_access(method, op->profiled_bci(), md, data, mdo_offset_bias);
2394 
2395     Label not_null;
2396     __ br_notnull_short(obj, Assembler::pn, not_null);
2397     Register mdo      = k_RInfo;
2398     Register data_val = Rtmp1;
2399     metadata2reg(md->constant_encoding(), mdo);
2400     if (mdo_offset_bias > 0) {
2401       __ set(mdo_offset_bias, data_val);
2402       __ add(mdo, data_val, mdo);
2403     }
2404     Address flags_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias);
2405     __ ldub(flags_addr, data_val);
2406     __ or3(data_val, BitData::null_seen_byte_constant(), data_val);
2407     __ stb(data_val, flags_addr);
2408     __ ba(*obj_is_null);
2409     __ delayed()->nop();
2410     __ bind(not_null);
2411   } else {
2412     __ br_null(obj, false, Assembler::pn, *obj_is_null);
2413     __ delayed()->nop();
2414   }
2415 
2416   Label profile_cast_failure, profile_cast_success;
2417   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
2418   Label *success_target = op->should_profile() ? &profile_cast_success : success;
2419 
2420   // patching may screw with our temporaries on sparc,
2421   // so let's do it before loading the class
2422   if (k->is_loaded()) {
2423     metadata2reg(k->constant_encoding(), k_RInfo);
2424   } else {
2425     klass2reg_with_patching(k_RInfo, op->info_for_patch());
2426   }
2427   assert(obj != k_RInfo, "must be different");
2428 
2429   // get object class
2430   // not a safepoint as obj null check happens earlier
2431   __ load_klass(obj, klass_RInfo);
2432   if (op->fast_check()) {
2433     assert_different_registers(klass_RInfo, k_RInfo);
2434     __ cmp(k_RInfo, klass_RInfo);
2435     __ brx(Assembler::notEqual, false, Assembler::pt, *failure_target);
2436     __ delayed()->nop();
2437   } else {
2438     bool need_slow_path = true;
2439     if (k->is_loaded()) {
2440       if ((int) k->super_check_offset() != in_bytes(Klass::secondary_super_cache_offset()))
2441         need_slow_path = false;
2442       // perform the fast part of the checking logic
2443       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, noreg,
2444                                        (need_slow_path ? success_target : NULL),
2445                                        failure_target, NULL,
2446                                        RegisterOrConstant(k->super_check_offset()));
2447     } else {
2448       // perform the fast part of the checking logic
2449       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, O7, success_target,
2450                                        failure_target, NULL);
2451     }
2452     if (need_slow_path) {
2453       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
2454       assert(klass_RInfo == G3 && k_RInfo == G1, "incorrect call setup");
2455       __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
2456       __ delayed()->nop();
2457       __ cmp(G3, 0);
2458       __ br(Assembler::equal, false, Assembler::pn, *failure_target);
2459       __ delayed()->nop();
2460       // Fall through to success case
2461     }
2462   }
2463 
2464   if (op->should_profile()) {
2465     Register mdo  = klass_RInfo, recv = k_RInfo, tmp1 = Rtmp1;
2466     assert_different_registers(obj, mdo, recv, tmp1);
2467     __ bind(profile_cast_success);
2468     metadata2reg(md->constant_encoding(), mdo);
2469     if (mdo_offset_bias > 0) {
2470       __ set(mdo_offset_bias, tmp1);
2471       __ add(mdo, tmp1, mdo);
2472     }
2473     __ load_klass(obj, recv);
2474     type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, success);
2475     // Jump over the failure case
2476     __ ba(*success);
2477     __ delayed()->nop();
2478     // Cast failure case
2479     __ bind(profile_cast_failure);
2480     metadata2reg(md->constant_encoding(), mdo);
2481     if (mdo_offset_bias > 0) {
2482       __ set(mdo_offset_bias, tmp1);
2483       __ add(mdo, tmp1, mdo);
2484     }
2485     Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
2486     __ ld_ptr(data_addr, tmp1);
2487     __ sub(tmp1, DataLayout::counter_increment, tmp1);
2488     __ st_ptr(tmp1, data_addr);
2489     __ ba(*failure);
2490     __ delayed()->nop();
2491   }
2492   __ ba(*success);
2493   __ delayed()->nop();
2494 }
2495 
2496 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
2497   LIR_Code code = op->code();
2498   if (code == lir_store_check) {
2499     Register value = op->object()->as_register();
2500     Register array = op->array()->as_register();
2501     Register k_RInfo = op->tmp1()->as_register();
2502     Register klass_RInfo = op->tmp2()->as_register();
2503     Register Rtmp1 = op->tmp3()->as_register();
2504 
2505     __ verify_oop(value);
2506     CodeStub* stub = op->stub();
2507     // check if it needs to be profiled
2508     ciMethodData* md;
2509     ciProfileData* data;
2510     int mdo_offset_bias = 0;
2511     if (op->should_profile()) {
2512       ciMethod* method = op->profiled_method();
2513       assert(method != NULL, "Should have method");
2514       setup_md_access(method, op->profiled_bci(), md, data, mdo_offset_bias);
2515     }
2516     Label profile_cast_success, profile_cast_failure, done;
2517     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
2518     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
2519 
2520     if (op->should_profile()) {
2521       Label not_null;
2522       __ br_notnull_short(value, Assembler::pn, not_null);
2523       Register mdo      = k_RInfo;
2524       Register data_val = Rtmp1;
2525       metadata2reg(md->constant_encoding(), mdo);
2526       if (mdo_offset_bias > 0) {
2527         __ set(mdo_offset_bias, data_val);
2528         __ add(mdo, data_val, mdo);
2529       }
2530       Address flags_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias);
2531       __ ldub(flags_addr, data_val);
2532       __ or3(data_val, BitData::null_seen_byte_constant(), data_val);
2533       __ stb(data_val, flags_addr);
2534       __ ba_short(done);
2535       __ bind(not_null);
2536     } else {
2537       __ br_null_short(value, Assembler::pn, done);
2538     }
2539     add_debug_info_for_null_check_here(op->info_for_exception());
2540     __ load_klass(array, k_RInfo);
2541     __ load_klass(value, klass_RInfo);
2542 
2543     // get instance klass
2544     __ ld_ptr(Address(k_RInfo, ObjArrayKlass::element_klass_offset()), k_RInfo);
2545     // perform the fast part of the checking logic
2546     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, O7, success_target, failure_target, NULL);
2547 
2548     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
2549     assert(klass_RInfo == G3 && k_RInfo == G1, "incorrect call setup");
2550     __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
2551     __ delayed()->nop();
2552     __ cmp(G3, 0);
2553     __ br(Assembler::equal, false, Assembler::pn, *failure_target);
2554     __ delayed()->nop();
2555     // fall through to the success case
2556 
2557     if (op->should_profile()) {
2558       Register mdo  = klass_RInfo, recv = k_RInfo, tmp1 = Rtmp1;
2559       assert_different_registers(value, mdo, recv, tmp1);
2560       __ bind(profile_cast_success);
2561       metadata2reg(md->constant_encoding(), mdo);
2562       if (mdo_offset_bias > 0) {
2563         __ set(mdo_offset_bias, tmp1);
2564         __ add(mdo, tmp1, mdo);
2565       }
2566       __ load_klass(value, recv);
2567       type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &done);
2568       __ ba_short(done);
2569       // Cast failure case
2570       __ bind(profile_cast_failure);
2571       metadata2reg(md->constant_encoding(), mdo);
2572       if (mdo_offset_bias > 0) {
2573         __ set(mdo_offset_bias, tmp1);
2574         __ add(mdo, tmp1, mdo);
2575       }
2576       Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
2577       __ ld_ptr(data_addr, tmp1);
2578       __ sub(tmp1, DataLayout::counter_increment, tmp1);
2579       __ st_ptr(tmp1, data_addr);
2580       __ ba(*stub->entry());
2581       __ delayed()->nop();
2582     }
2583     __ bind(done);
2584   } else if (code == lir_checkcast) {
2585     Register obj = op->object()->as_register();
2586     Register dst = op->result_opr()->as_register();
2587     Label success;
2588     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
2589     __ bind(success);
2590     __ mov(obj, dst);
2591   } else if (code == lir_instanceof) {
2592     Register obj = op->object()->as_register();
2593     Register dst = op->result_opr()->as_register();
2594     Label success, failure, done;
2595     emit_typecheck_helper(op, &success, &failure, &failure);
2596     __ bind(failure);
2597     __ set(0, dst);
2598     __ ba_short(done);
2599     __ bind(success);
2600     __ set(1, dst);
2601     __ bind(done);
2602   } else {
2603     ShouldNotReachHere();
2604   }
2605 
2606 }
2607 
2608 
2609 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2610   if (op->code() == lir_cas_long) {
2611     assert(VM_Version::supports_cx8(), "wrong machine");
2612     Register addr = op->addr()->as_pointer_register();
2613     Register cmp_value_lo = op->cmp_value()->as_register_lo();
2614     Register cmp_value_hi = op->cmp_value()->as_register_hi();
2615     Register new_value_lo = op->new_value()->as_register_lo();
2616     Register new_value_hi = op->new_value()->as_register_hi();
2617     Register t1 = op->tmp1()->as_register();
2618     Register t2 = op->tmp2()->as_register();
2619     __ mov(cmp_value_lo, t1);
2620     __ mov(new_value_lo, t2);
2621     // perform the compare and swap operation
2622     __ casx(addr, t1, t2);
2623     // generate condition code - if the swap succeeded, t2 ("new value" reg) was
2624     // overwritten with the original value in "addr" and will be equal to t1.
2625     __ cmp(t1, t2);
2626   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
2627     Register addr = op->addr()->as_pointer_register();
2628     Register cmp_value = op->cmp_value()->as_register();
2629     Register new_value = op->new_value()->as_register();
2630     Register t1 = op->tmp1()->as_register();
2631     Register t2 = op->tmp2()->as_register();
2632     __ mov(cmp_value, t1);
2633     __ mov(new_value, t2);
2634     if (op->code() == lir_cas_obj) {
2635       if (UseCompressedOops) {
2636         __ encode_heap_oop(t1);
2637         __ encode_heap_oop(t2);
2638         __ cas(addr, t1, t2);
2639       } else {
2640         __ cas_ptr(addr, t1, t2);
2641       }
2642     } else {
2643       __ cas(addr, t1, t2);
2644     }
2645     __ cmp(t1, t2);
2646   } else {
2647     Unimplemented();
2648   }
2649 }
2650 
2651 void LIR_Assembler::set_24bit_FPU() {
2652   Unimplemented();
2653 }
2654 
2655 
2656 void LIR_Assembler::reset_FPU() {
2657   Unimplemented();
2658 }
2659 
2660 
2661 void LIR_Assembler::breakpoint() {
2662   __ breakpoint_trap();
2663 }
2664 
2665 
2666 void LIR_Assembler::push(LIR_Opr opr) {
2667   Unimplemented();
2668 }
2669 
2670 
2671 void LIR_Assembler::pop(LIR_Opr opr) {
2672   Unimplemented();
2673 }
2674 
2675 
2676 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst_opr) {
2677   Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no);
2678   Register dst = dst_opr->as_register();
2679   Register reg = mon_addr.base();
2680   int offset = mon_addr.disp();
2681   // compute pointer to BasicLock
2682   if (mon_addr.is_simm13()) {
2683     __ add(reg, offset, dst);
2684   } else {
2685     __ set(offset, dst);
2686     __ add(dst, reg, dst);
2687   }
2688 }
2689 
2690 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2691   assert(op->crc()->is_single_cpu(),  "crc must be register");
2692   assert(op->val()->is_single_cpu(),  "byte value must be register");
2693   assert(op->result_opr()->is_single_cpu(), "result must be register");
2694   Register crc = op->crc()->as_register();
2695   Register val = op->val()->as_register();
2696   Register table = op->result_opr()->as_register();
2697   Register res   = op->result_opr()->as_register();
2698 
2699   assert_different_registers(val, crc, table);
2700 
2701   __ set(ExternalAddress(StubRoutines::crc_table_addr()), table);
2702   __ not1(crc);
2703   __ clruwu(crc);
2704   __ update_byte_crc32(crc, val, table);
2705   __ not1(crc);
2706 
2707   __ mov(crc, res);
2708 }
2709 
2710 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2711   Register obj = op->obj_opr()->as_register();
2712   Register hdr = op->hdr_opr()->as_register();
2713   Register lock = op->lock_opr()->as_register();
2714 
2715   // obj may not be an oop
2716   if (op->code() == lir_lock) {
2717     MonitorEnterStub* stub = (MonitorEnterStub*)op->stub();
2718     if (UseFastLocking) {
2719       assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2720       // add debug info for NullPointerException only if one is possible
2721       if (op->info() != NULL) {
2722         add_debug_info_for_null_check_here(op->info());
2723       }
2724       __ lock_object(hdr, obj, lock, op->scratch_opr()->as_register(), *op->stub()->entry());
2725     } else {
2726       // always do slow locking
2727       // note: the slow locking code could be inlined here, however if we use
2728       //       slow locking, speed doesn't matter anyway and this solution is
2729       //       simpler and requires less duplicated code - additionally, the
2730       //       slow locking code is the same in either case which simplifies
2731       //       debugging
2732       __ br(Assembler::always, false, Assembler::pt, *op->stub()->entry());
2733       __ delayed()->nop();
2734     }
2735   } else {
2736     assert (op->code() == lir_unlock, "Invalid code, expected lir_unlock");
2737     if (UseFastLocking) {
2738       assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2739       __ unlock_object(hdr, obj, lock, *op->stub()->entry());
2740     } else {
2741       // always do slow unlocking
2742       // note: the slow unlocking code could be inlined here, however if we use
2743       //       slow unlocking, speed doesn't matter anyway and this solution is
2744       //       simpler and requires less duplicated code - additionally, the
2745       //       slow unlocking code is the same in either case which simplifies
2746       //       debugging
2747       __ br(Assembler::always, false, Assembler::pt, *op->stub()->entry());
2748       __ delayed()->nop();
2749     }
2750   }
2751   __ bind(*op->stub()->continuation());
2752 }
2753 
2754 
2755 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
2756   ciMethod* method = op->profiled_method();
2757   int bci          = op->profiled_bci();
2758   ciMethod* callee = op->profiled_callee();
2759 
2760   // Update counter for all call types
2761   ciMethodData* md = method->method_data_or_null();
2762   assert(md != NULL, "Sanity");
2763   ciProfileData* data = md->bci_to_data(bci);
2764   assert(data != NULL && data->is_CounterData(), "need CounterData for calls");
2765   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
2766   Register mdo  = op->mdo()->as_register();
2767   assert(op->tmp1()->is_double_cpu(), "tmp1 must be allocated");
2768   Register tmp1 = op->tmp1()->as_register_lo();
2769   metadata2reg(md->constant_encoding(), mdo);
2770   int mdo_offset_bias = 0;
2771   if (!Assembler::is_simm13(md->byte_offset_of_slot(data, CounterData::count_offset()) +
2772                             data->size_in_bytes())) {
2773     // The offset is large so bias the mdo by the base of the slot so
2774     // that the ld can use simm13s to reference the slots of the data
2775     mdo_offset_bias = md->byte_offset_of_slot(data, CounterData::count_offset());
2776     __ set(mdo_offset_bias, O7);
2777     __ add(mdo, O7, mdo);
2778   }
2779 
2780   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
2781   // Perform additional virtual call profiling for invokevirtual and
2782   // invokeinterface bytecodes
2783   if (op->should_profile_receiver_type()) {
2784     assert(op->recv()->is_single_cpu(), "recv must be allocated");
2785     Register recv = op->recv()->as_register();
2786     assert_different_registers(mdo, tmp1, recv);
2787     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
2788     ciKlass* known_klass = op->known_holder();
2789     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
2790       // We know the type that will be seen at this call site; we can
2791       // statically update the MethodData* rather than needing to do
2792       // dynamic tests on the receiver type
2793 
2794       // NOTE: we should probably put a lock around this search to
2795       // avoid collisions by concurrent compilations
2796       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
2797       uint i;
2798       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2799         ciKlass* receiver = vc_data->receiver(i);
2800         if (known_klass->equals(receiver)) {
2801           Address data_addr(mdo, md->byte_offset_of_slot(data,
2802                                                          VirtualCallData::receiver_count_offset(i)) -
2803                             mdo_offset_bias);
2804           __ ld_ptr(data_addr, tmp1);
2805           __ add(tmp1, DataLayout::counter_increment, tmp1);
2806           __ st_ptr(tmp1, data_addr);
2807           return;
2808         }
2809       }
2810 
2811       // Receiver type not found in profile data; select an empty slot
2812 
2813       // Note that this is less efficient than it should be because it
2814       // always does a write to the receiver part of the
2815       // VirtualCallData rather than just the first time
2816       for (i = 0; i < VirtualCallData::row_limit(); i++) {
2817         ciKlass* receiver = vc_data->receiver(i);
2818         if (receiver == NULL) {
2819           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)) -
2820                             mdo_offset_bias);
2821           metadata2reg(known_klass->constant_encoding(), tmp1);
2822           __ st_ptr(tmp1, recv_addr);
2823           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)) -
2824                             mdo_offset_bias);
2825           __ ld_ptr(data_addr, tmp1);
2826           __ add(tmp1, DataLayout::counter_increment, tmp1);
2827           __ st_ptr(tmp1, data_addr);
2828           return;
2829         }
2830       }
2831     } else {
2832       __ load_klass(recv, recv);
2833       Label update_done;
2834       type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &update_done);
2835       // Receiver did not match any saved receiver and there is no empty row for it.
2836       // Increment total counter to indicate polymorphic case.
2837       __ ld_ptr(counter_addr, tmp1);
2838       __ add(tmp1, DataLayout::counter_increment, tmp1);
2839       __ st_ptr(tmp1, counter_addr);
2840 
2841       __ bind(update_done);
2842     }
2843   } else {
2844     // Static call
2845     __ ld_ptr(counter_addr, tmp1);
2846     __ add(tmp1, DataLayout::counter_increment, tmp1);
2847     __ st_ptr(tmp1, counter_addr);
2848   }
2849 }
2850 
2851 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
2852   Register obj = op->obj()->as_register();
2853   Register tmp1 = op->tmp()->as_pointer_register();
2854   Register tmp2 = G1;
2855   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
2856   ciKlass* exact_klass = op->exact_klass();
2857   intptr_t current_klass = op->current_klass();
2858   bool not_null = op->not_null();
2859   bool no_conflict = op->no_conflict();
2860 
2861   Label update, next, none;
2862 
2863   bool do_null = !not_null;
2864   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
2865   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
2866 
2867   assert(do_null || do_update, "why are we here?");
2868   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
2869 
2870   __ verify_oop(obj);
2871 
2872   if (tmp1 != obj) {
2873     __ mov(obj, tmp1);
2874   }
2875   if (do_null) {
2876     __ br_notnull_short(tmp1, Assembler::pt, update);
2877     if (!TypeEntries::was_null_seen(current_klass)) {
2878       __ ld_ptr(mdo_addr, tmp1);
2879       __ or3(tmp1, TypeEntries::null_seen, tmp1);
2880       __ st_ptr(tmp1, mdo_addr);
2881     }
2882     if (do_update) {
2883       __ ba(next);
2884       __ delayed()->nop();
2885     }
2886 #ifdef ASSERT
2887   } else {
2888     __ br_notnull_short(tmp1, Assembler::pt, update);
2889     __ stop("unexpect null obj");
2890 #endif
2891   }
2892 
2893   __ bind(update);
2894 
2895   if (do_update) {
2896 #ifdef ASSERT
2897     if (exact_klass != NULL) {
2898       Label ok;
2899       __ load_klass(tmp1, tmp1);
2900       metadata2reg(exact_klass->constant_encoding(), tmp2);
2901       __ cmp_and_br_short(tmp1, tmp2, Assembler::equal, Assembler::pt, ok);
2902       __ stop("exact klass and actual klass differ");
2903       __ bind(ok);
2904     }
2905 #endif
2906 
2907     Label do_update;
2908     __ ld_ptr(mdo_addr, tmp2);
2909 
2910     if (!no_conflict) {
2911       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
2912         if (exact_klass != NULL) {
2913           metadata2reg(exact_klass->constant_encoding(), tmp1);
2914         } else {
2915           __ load_klass(tmp1, tmp1);
2916         }
2917 
2918         __ xor3(tmp1, tmp2, tmp1);
2919         __ btst(TypeEntries::type_klass_mask, tmp1);
2920         // klass seen before, nothing to do. The unknown bit may have been
2921         // set already but no need to check.
2922         __ brx(Assembler::zero, false, Assembler::pt, next);
2923         __ delayed()->
2924 
2925            btst(TypeEntries::type_unknown, tmp1);
2926         // already unknown. Nothing to do anymore.
2927         __ brx(Assembler::notZero, false, Assembler::pt, next);
2928 
2929         if (TypeEntries::is_type_none(current_klass)) {
2930           __ delayed()->btst(TypeEntries::type_mask, tmp2);
2931           __ brx(Assembler::zero, true, Assembler::pt, do_update);
2932           // first time here. Set profile type.
2933           __ delayed()->or3(tmp2, tmp1, tmp2);
2934         } else {
2935           __ delayed()->nop();
2936         }
2937       } else {
2938         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2939                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
2940 
2941         __ btst(TypeEntries::type_unknown, tmp2);
2942         // already unknown. Nothing to do anymore.
2943         __ brx(Assembler::notZero, false, Assembler::pt, next);
2944         __ delayed()->nop();
2945       }
2946 
2947       // different than before. Cannot keep accurate profile.
2948       __ or3(tmp2, TypeEntries::type_unknown, tmp2);
2949     } else {
2950       // There's a single possible klass at this profile point
2951       assert(exact_klass != NULL, "should be");
2952       if (TypeEntries::is_type_none(current_klass)) {
2953         metadata2reg(exact_klass->constant_encoding(), tmp1);
2954         __ xor3(tmp1, tmp2, tmp1);
2955         __ btst(TypeEntries::type_klass_mask, tmp1);
2956         __ brx(Assembler::zero, false, Assembler::pt, next);
2957 #ifdef ASSERT
2958 
2959         {
2960           Label ok;
2961           __ delayed()->btst(TypeEntries::type_mask, tmp2);
2962           __ brx(Assembler::zero, true, Assembler::pt, ok);
2963           __ delayed()->nop();
2964 
2965           __ stop("unexpected profiling mismatch");
2966           __ bind(ok);
2967         }
2968         // first time here. Set profile type.
2969         __ or3(tmp2, tmp1, tmp2);
2970 #else
2971         // first time here. Set profile type.
2972         __ delayed()->or3(tmp2, tmp1, tmp2);
2973 #endif
2974 
2975       } else {
2976         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
2977                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
2978 
2979         // already unknown. Nothing to do anymore.
2980         __ btst(TypeEntries::type_unknown, tmp2);
2981         __ brx(Assembler::notZero, false, Assembler::pt, next);
2982         __ delayed()->or3(tmp2, TypeEntries::type_unknown, tmp2);
2983       }
2984     }
2985 
2986     __ bind(do_update);
2987     __ st_ptr(tmp2, mdo_addr);
2988 
2989     __ bind(next);
2990   }
2991 }
2992 
2993 void LIR_Assembler::align_backward_branch_target() {
2994   __ align(OptoLoopAlignment);
2995 }
2996 
2997 
2998 void LIR_Assembler::emit_delay(LIR_OpDelay* op) {
2999   // make sure we are expecting a delay
3000   // this has the side effect of clearing the delay state
3001   // so we can use _masm instead of _masm->delayed() to do the
3002   // code generation.
3003   __ delayed();
3004 
3005   // make sure we only emit one instruction
3006   int offset = code_offset();
3007   op->delay_op()->emit_code(this);
3008 #ifdef ASSERT
3009   if (code_offset() - offset != NativeInstruction::nop_instruction_size) {
3010     op->delay_op()->print();
3011   }
3012   assert(code_offset() - offset == NativeInstruction::nop_instruction_size,
3013          "only one instruction can go in a delay slot");
3014 #endif
3015 
3016   // we may also be emitting the call info for the instruction
3017   // which we are the delay slot of.
3018   CodeEmitInfo* call_info = op->call_info();
3019   if (call_info) {
3020     add_call_info(code_offset(), call_info);
3021   }
3022 
3023   if (VerifyStackAtCalls) {
3024     _masm->sub(FP, SP, O7);
3025     _masm->cmp(O7, initial_frame_size_in_bytes());
3026     _masm->trap(Assembler::notEqual, Assembler::ptr_cc, G0, ST_RESERVED_FOR_USER_0+2 );
3027   }
3028 }
3029 
3030 
3031 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3032   // tmp must be unused
3033   assert(tmp->is_illegal(), "wasting a register if tmp is allocated");
3034   assert(left->is_register(), "can only handle registers");
3035 
3036   if (left->is_single_cpu()) {
3037     __ neg(left->as_register(), dest->as_register());
3038   } else if (left->is_single_fpu()) {
3039     __ fneg(FloatRegisterImpl::S, left->as_float_reg(), dest->as_float_reg());
3040   } else if (left->is_double_fpu()) {
3041     __ fneg(FloatRegisterImpl::D, left->as_double_reg(), dest->as_double_reg());
3042   } else {
3043     assert (left->is_double_cpu(), "Must be a long");
3044     Register Rlow = left->as_register_lo();
3045     Register Rhi = left->as_register_hi();
3046     __ sub(G0, Rlow, dest->as_register_lo());
3047   }
3048 }
3049 
3050 
3051 void LIR_Assembler::fxch(int i) {
3052   Unimplemented();
3053 }
3054 
3055 void LIR_Assembler::fld(int i) {
3056   Unimplemented();
3057 }
3058 
3059 void LIR_Assembler::ffree(int i) {
3060   Unimplemented();
3061 }
3062 
3063 void LIR_Assembler::rt_call(LIR_Opr result, address dest,
3064                             const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3065 
3066   // if tmp is invalid, then the function being called doesn't destroy the thread
3067   if (tmp->is_valid()) {
3068     __ save_thread(tmp->as_pointer_register());
3069   }
3070   __ call(dest, relocInfo::runtime_call_type);
3071   __ delayed()->nop();
3072   if (info != NULL) {
3073     add_call_info_here(info);
3074   }
3075   if (tmp->is_valid()) {
3076     __ restore_thread(tmp->as_pointer_register());
3077   }
3078 
3079 #ifdef ASSERT
3080   __ verify_thread();
3081 #endif // ASSERT
3082 }
3083 
3084 
3085 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3086   ShouldNotReachHere();
3087 
3088   NEEDS_CLEANUP;
3089   if (type == T_LONG) {
3090     LIR_Address* mem_addr = dest->is_address() ? dest->as_address_ptr() : src->as_address_ptr();
3091 
3092     // (extended to allow indexed as well as constant displaced for JSR-166)
3093     Register idx = noreg; // contains either constant offset or index
3094 
3095     int disp = mem_addr->disp();
3096     if (mem_addr->index() == LIR_OprFact::illegalOpr) {
3097       if (!Assembler::is_simm13(disp)) {
3098         idx = O7;
3099         __ set(disp, idx);
3100       }
3101     } else {
3102       assert(disp == 0, "not both indexed and disp");
3103       idx = mem_addr->index()->as_register();
3104     }
3105 
3106     int null_check_offset = -1;
3107 
3108     Register base = mem_addr->base()->as_register();
3109     if (src->is_register() && dest->is_address()) {
3110       // G4 is high half, G5 is low half
3111       // clear the top bits of G5, and scale up G4
3112       __ srl (src->as_register_lo(),  0, G5);
3113       __ sllx(src->as_register_hi(), 32, G4);
3114       // combine the two halves into the 64 bits of G4
3115       __ or3(G4, G5, G4);
3116       null_check_offset = __ offset();
3117       if (idx == noreg) {
3118         __ stx(G4, base, disp);
3119       } else {
3120         __ stx(G4, base, idx);
3121       }
3122     } else if (src->is_address() && dest->is_register()) {
3123       null_check_offset = __ offset();
3124       if (idx == noreg) {
3125         __ ldx(base, disp, G5);
3126       } else {
3127         __ ldx(base, idx, G5);
3128       }
3129       __ srax(G5, 32, dest->as_register_hi()); // fetch the high half into hi
3130       __ mov (G5, dest->as_register_lo());     // copy low half into lo
3131     } else {
3132       Unimplemented();
3133     }
3134     if (info != NULL) {
3135       add_debug_info_for_null_check(null_check_offset, info);
3136     }
3137 
3138   } else {
3139     // use normal move for all other volatiles since they don't need
3140     // special handling to remain atomic.
3141     move_op(src, dest, type, lir_patch_none, info, false, false, false);
3142   }
3143 }
3144 
3145 void LIR_Assembler::membar() {
3146   // only StoreLoad membars are ever explicitly needed on sparcs in TSO mode
3147   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad) );
3148 }
3149 
3150 void LIR_Assembler::membar_acquire() {
3151   // no-op on TSO
3152 }
3153 
3154 void LIR_Assembler::membar_release() {
3155   // no-op on TSO
3156 }
3157 
3158 void LIR_Assembler::membar_loadload() {
3159   // no-op
3160   //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3161 }
3162 
3163 void LIR_Assembler::membar_storestore() {
3164   // no-op
3165   //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3166 }
3167 
3168 void LIR_Assembler::membar_loadstore() {
3169   // no-op
3170   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3171 }
3172 
3173 void LIR_Assembler::membar_storeload() {
3174   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3175 }
3176 
3177 void LIR_Assembler::on_spin_wait() {
3178   Unimplemented();
3179 }
3180 
3181 // Pack two sequential registers containing 32 bit values
3182 // into a single 64 bit register.
3183 // src and src->successor() are packed into dst
3184 // src and dst may be the same register.
3185 // Note: src is destroyed
3186 void LIR_Assembler::pack64(LIR_Opr src, LIR_Opr dst) {
3187   Register rs = src->as_register();
3188   Register rd = dst->as_register_lo();
3189   __ sllx(rs, 32, rs);
3190   __ srl(rs->successor(), 0, rs->successor());
3191   __ or3(rs, rs->successor(), rd);
3192 }
3193 
3194 // Unpack a 64 bit value in a register into
3195 // two sequential registers.
3196 // src is unpacked into dst and dst->successor()
3197 void LIR_Assembler::unpack64(LIR_Opr src, LIR_Opr dst) {
3198   Register rs = src->as_register_lo();
3199   Register rd = dst->as_register_hi();
3200   assert_different_registers(rs, rd, rd->successor());
3201   __ srlx(rs, 32, rd);
3202   __ srl (rs,  0, rd->successor());
3203 }
3204 
3205 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
3206   const LIR_Address* addr = addr_opr->as_address_ptr();
3207   assert(addr->scale() == LIR_Address::times_1, "can't handle complex addresses yet");
3208   const Register dest_reg = dest->as_pointer_register();
3209   const Register base_reg = addr->base()->as_pointer_register();
3210 
3211   if (patch_code != lir_patch_none) {
3212     PatchingStub* patch = new PatchingStub(_masm, PatchingStub::access_field_id);
3213     assert(addr->disp() != 0, "must have");
3214     assert(base_reg != G3_scratch, "invariant");
3215     __ patchable_set(0, G3_scratch);
3216     patching_epilog(patch, patch_code, base_reg, info);
3217     assert(dest_reg != G3_scratch, "invariant");
3218     if (addr->index()->is_valid()) {
3219       const Register index_reg = addr->index()->as_pointer_register();
3220       assert(index_reg != G3_scratch, "invariant");
3221       __ add(index_reg, G3_scratch, G3_scratch);
3222     }
3223     __ add(base_reg, G3_scratch, dest_reg);
3224   } else {
3225     if (Assembler::is_simm13(addr->disp())) {
3226       if (addr->index()->is_valid()) {
3227         const Register index_reg = addr->index()->as_pointer_register();
3228         assert(index_reg != G3_scratch, "invariant");
3229         __ add(base_reg, addr->disp(), G3_scratch);
3230         __ add(index_reg, G3_scratch, dest_reg);
3231       } else {
3232         __ add(base_reg, addr->disp(), dest_reg);
3233       }
3234     } else {
3235       __ set(addr->disp(), G3_scratch);
3236       if (addr->index()->is_valid()) {
3237         const Register index_reg = addr->index()->as_pointer_register();
3238         assert(index_reg != G3_scratch, "invariant");
3239         __ add(index_reg, G3_scratch, G3_scratch);
3240       }
3241       __ add(base_reg, G3_scratch, dest_reg);
3242     }
3243   }
3244 }
3245 
3246 
3247 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3248   assert(result_reg->is_register(), "check");
3249   __ mov(G2_thread, result_reg->as_register());
3250 }
3251 
3252 #ifdef ASSERT
3253 // emit run-time assertion
3254 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3255   assert(op->code() == lir_assert, "must be");
3256 
3257   if (op->in_opr1()->is_valid()) {
3258     assert(op->in_opr2()->is_valid(), "both operands must be valid");
3259     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3260   } else {
3261     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3262     assert(op->condition() == lir_cond_always, "no other conditions allowed");
3263   }
3264 
3265   Label ok;
3266   if (op->condition() != lir_cond_always) {
3267     Assembler::Condition acond;
3268     switch (op->condition()) {
3269       case lir_cond_equal:        acond = Assembler::equal;                break;
3270       case lir_cond_notEqual:     acond = Assembler::notEqual;             break;
3271       case lir_cond_less:         acond = Assembler::less;                 break;
3272       case lir_cond_lessEqual:    acond = Assembler::lessEqual;            break;
3273       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;         break;
3274       case lir_cond_greater:      acond = Assembler::greater;              break;
3275       case lir_cond_aboveEqual:   acond = Assembler::greaterEqualUnsigned; break;
3276       case lir_cond_belowEqual:   acond = Assembler::lessEqualUnsigned;    break;
3277       default:                         ShouldNotReachHere();
3278     };
3279     __ br(acond, false, Assembler::pt, ok);
3280     __ delayed()->nop();
3281   }
3282   if (op->halt()) {
3283     const char* str = __ code_string(op->msg());
3284     __ stop(str);
3285   } else {
3286     breakpoint();
3287   }
3288   __ bind(ok);
3289 }
3290 #endif
3291 
3292 void LIR_Assembler::peephole(LIR_List* lir) {
3293   LIR_OpList* inst = lir->instructions_list();
3294   for (int i = 0; i < inst->length(); i++) {
3295     LIR_Op* op = inst->at(i);
3296     switch (op->code()) {
3297       case lir_cond_float_branch:
3298       case lir_branch: {
3299         LIR_OpBranch* branch = op->as_OpBranch();
3300         assert(branch->info() == NULL, "shouldn't be state on branches anymore");
3301         LIR_Op* delay_op = NULL;
3302         // we'd like to be able to pull following instructions into
3303         // this slot but we don't know enough to do it safely yet so
3304         // only optimize block to block control flow.
3305         if (LIRFillDelaySlots && branch->block()) {
3306           LIR_Op* prev = inst->at(i - 1);
3307           if (prev && LIR_Assembler::is_single_instruction(prev) && prev->info() == NULL) {
3308             // swap previous instruction into delay slot
3309             inst->at_put(i - 1, op);
3310             inst->at_put(i, new LIR_OpDelay(prev, op->info()));
3311 #ifndef PRODUCT
3312             if (LIRTracePeephole) {
3313               tty->print_cr("delayed");
3314               inst->at(i - 1)->print();
3315               inst->at(i)->print();
3316               tty->cr();
3317             }
3318 #endif
3319             continue;
3320           }
3321         }
3322 
3323         if (!delay_op) {
3324           delay_op = new LIR_OpDelay(new LIR_Op0(lir_nop), NULL);
3325         }
3326         inst->insert_before(i + 1, delay_op);
3327         break;
3328       }
3329       case lir_static_call:
3330       case lir_virtual_call:
3331       case lir_icvirtual_call:
3332       case lir_optvirtual_call:
3333       case lir_dynamic_call: {
3334         LIR_Op* prev = inst->at(i - 1);
3335         if (LIRFillDelaySlots && prev && prev->code() == lir_move && prev->info() == NULL &&
3336             (op->code() != lir_virtual_call ||
3337              !prev->result_opr()->is_single_cpu() ||
3338              prev->result_opr()->as_register() != O0) &&
3339             LIR_Assembler::is_single_instruction(prev)) {
3340           // Only moves without info can be put into the delay slot.
3341           // Also don't allow the setup of the receiver in the delay
3342           // slot for vtable calls.
3343           inst->at_put(i - 1, op);
3344           inst->at_put(i, new LIR_OpDelay(prev, op->info()));
3345 #ifndef PRODUCT
3346           if (LIRTracePeephole) {
3347             tty->print_cr("delayed");
3348             inst->at(i - 1)->print();
3349             inst->at(i)->print();
3350             tty->cr();
3351           }
3352 #endif
3353         } else {
3354           LIR_Op* delay_op = new LIR_OpDelay(new LIR_Op0(lir_nop), op->as_OpJavaCall()->info());
3355           inst->insert_before(i + 1, delay_op);
3356           i++;
3357         }
3358         break;
3359       }
3360     }
3361   }
3362 }
3363 
3364 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3365   LIR_Address* addr = src->as_address_ptr();
3366 
3367   assert(data == dest, "swap uses only 2 operands");
3368   assert (code == lir_xchg, "no xadd on sparc");
3369 
3370   if (data->type() == T_INT) {
3371     __ swap(as_Address(addr), data->as_register());
3372   } else if (data->is_oop()) {
3373     Register obj = data->as_register();
3374     Register narrow = tmp->as_register();
3375     assert(UseCompressedOops, "swap is 32bit only");
3376     __ encode_heap_oop(obj, narrow);
3377     __ swap(as_Address(addr), narrow);
3378     __ decode_heap_oop(narrow, obj);
3379   } else {
3380     ShouldNotReachHere();
3381   }
3382 }
3383 
3384 #undef __