1 /*
   2  * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "c1/c1_Compilation.hpp"
  27 #include "c1/c1_LIRAssembler.hpp"
  28 #include "c1/c1_MacroAssembler.hpp"
  29 #include "c1/c1_Runtime1.hpp"
  30 #include "c1/c1_ValueStack.hpp"
  31 #include "ci/ciArrayKlass.hpp"
  32 #include "ci/ciInstance.hpp"
  33 #include "gc_interface/collectedHeap.hpp"
  34 #include "memory/barrierSet.hpp"
  35 #include "memory/cardTableModRefBS.hpp"
  36 #include "nativeInst_sparc.hpp"
  37 #include "oops/objArrayKlass.hpp"
  38 #include "runtime/sharedRuntime.hpp"
  39 
  40 #define __ _masm->
  41 
  42 
  43 //------------------------------------------------------------
  44 
  45 
  46 bool LIR_Assembler::is_small_constant(LIR_Opr opr) {
  47   if (opr->is_constant()) {
  48     LIR_Const* constant = opr->as_constant_ptr();
  49     switch (constant->type()) {
  50       case T_INT: {
  51         jint value = constant->as_jint();
  52         return Assembler::is_simm13(value);
  53       }
  54 
  55       default:
  56         return false;
  57     }
  58   }
  59   return false;
  60 }
  61 
  62 
  63 bool LIR_Assembler::is_single_instruction(LIR_Op* op) {
  64   switch (op->code()) {
  65     case lir_null_check:
  66     return true;
  67 
  68 
  69     case lir_add:
  70     case lir_ushr:
  71     case lir_shr:
  72     case lir_shl:
  73       // integer shifts and adds are always one instruction
  74       return op->result_opr()->is_single_cpu();
  75 
  76 
  77     case lir_move: {
  78       LIR_Op1* op1 = op->as_Op1();
  79       LIR_Opr src = op1->in_opr();
  80       LIR_Opr dst = op1->result_opr();
  81 
  82       if (src == dst) {
  83         NEEDS_CLEANUP;
  84         // this works around a problem where moves with the same src and dst
  85         // end up in the delay slot and then the assembler swallows the mov
  86         // since it has no effect and then it complains because the delay slot
  87         // is empty.  returning false stops the optimizer from putting this in
  88         // the delay slot
  89         return false;
  90       }
  91 
  92       // don't put moves involving oops into the delay slot since the VerifyOops code
  93       // will make it much larger than a single instruction.
  94       if (VerifyOops) {
  95         return false;
  96       }
  97 
  98       if (src->is_double_cpu() || dst->is_double_cpu() || op1->patch_code() != lir_patch_none ||
  99           ((src->is_double_fpu() || dst->is_double_fpu()) && op1->move_kind() != lir_move_normal)) {
 100         return false;
 101       }
 102 
 103       if (UseCompressedOops) {
 104         if (dst->is_address() && !dst->is_stack() && (dst->type() == T_OBJECT || dst->type() == T_ARRAY)) return false;
 105         if (src->is_address() && !src->is_stack() && (src->type() == T_OBJECT || src->type() == T_ARRAY)) return false;
 106       }
 107 
 108       if (UseCompressedClassPointers) {
 109         if (src->is_address() && !src->is_stack() && src->type() == T_ADDRESS &&
 110             src->as_address_ptr()->disp() == oopDesc::klass_offset_in_bytes()) return false;
 111       }
 112 
 113       if (dst->is_register()) {
 114         if (src->is_address() && Assembler::is_simm13(src->as_address_ptr()->disp())) {
 115           return !PatchALot;
 116         } else if (src->is_single_stack()) {
 117           return true;
 118         }
 119       }
 120 
 121       if (src->is_register()) {
 122         if (dst->is_address() && Assembler::is_simm13(dst->as_address_ptr()->disp())) {
 123           return !PatchALot;
 124         } else if (dst->is_single_stack()) {
 125           return true;
 126         }
 127       }
 128 
 129       if (dst->is_register() &&
 130           ((src->is_register() && src->is_single_word() && src->is_same_type(dst)) ||
 131            (src->is_constant() && LIR_Assembler::is_small_constant(op->as_Op1()->in_opr())))) {
 132         return true;
 133       }
 134 
 135       return false;
 136     }
 137 
 138     default:
 139       return false;
 140   }
 141   ShouldNotReachHere();
 142 }
 143 
 144 
 145 LIR_Opr LIR_Assembler::receiverOpr() {
 146   return FrameMap::O0_oop_opr;
 147 }
 148 
 149 
 150 LIR_Opr LIR_Assembler::osrBufferPointer() {
 151   return FrameMap::I0_opr;
 152 }
 153 
 154 
 155 int LIR_Assembler::initial_frame_size_in_bytes() const {
 156   return in_bytes(frame_map()->framesize_in_bytes());
 157 }
 158 
 159 
 160 // inline cache check: the inline cached class is in G5_inline_cache_reg(G5);
 161 // we fetch the class of the receiver (O0) and compare it with the cached class.
 162 // If they do not match we jump to slow case.
 163 int LIR_Assembler::check_icache() {
 164   int offset = __ offset();
 165   __ inline_cache_check(O0, G5_inline_cache_reg);
 166   return offset;
 167 }
 168 
 169 
 170 void LIR_Assembler::osr_entry() {
 171   // On-stack-replacement entry sequence (interpreter frame layout described in interpreter_sparc.cpp):
 172   //
 173   //   1. Create a new compiled activation.
 174   //   2. Initialize local variables in the compiled activation.  The expression stack must be empty
 175   //      at the osr_bci; it is not initialized.
 176   //   3. Jump to the continuation address in compiled code to resume execution.
 177 
 178   // OSR entry point
 179   offsets()->set_value(CodeOffsets::OSR_Entry, code_offset());
 180   BlockBegin* osr_entry = compilation()->hir()->osr_entry();
 181   ValueStack* entry_state = osr_entry->end()->state();
 182   int number_of_locks = entry_state->locks_size();
 183 
 184   // Create a frame for the compiled activation.
 185   __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes());
 186 
 187   // OSR buffer is
 188   //
 189   // locals[nlocals-1..0]
 190   // monitors[number_of_locks-1..0]
 191   //
 192   // locals is a direct copy of the interpreter frame so in the osr buffer
 193   // so first slot in the local array is the last local from the interpreter
 194   // and last slot is local[0] (receiver) from the interpreter
 195   //
 196   // Similarly with locks. The first lock slot in the osr buffer is the nth lock
 197   // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock
 198   // in the interpreter frame (the method lock if a sync method)
 199 
 200   // Initialize monitors in the compiled activation.
 201   //   I0: pointer to osr buffer
 202   //
 203   // All other registers are dead at this point and the locals will be
 204   // copied into place by code emitted in the IR.
 205 
 206   Register OSR_buf = osrBufferPointer()->as_register();
 207   { assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below");
 208     int monitor_offset = BytesPerWord * method()->max_locals() +
 209       (2 * BytesPerWord) * (number_of_locks - 1);
 210     // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in
 211     // the OSR buffer using 2 word entries: first the lock and then
 212     // the oop.
 213     for (int i = 0; i < number_of_locks; i++) {
 214       int slot_offset = monitor_offset - ((i * 2) * BytesPerWord);
 215 #ifdef ASSERT
 216       // verify the interpreter's monitor has a non-null object
 217       {
 218         Label L;
 219         __ ld_ptr(OSR_buf, slot_offset + 1*BytesPerWord, O7);
 220         __ cmp_and_br_short(O7, G0, Assembler::notEqual, Assembler::pt, L);
 221         __ stop("locked object is NULL");
 222         __ bind(L);
 223       }
 224 #endif // ASSERT
 225       // Copy the lock field into the compiled activation.
 226       __ ld_ptr(OSR_buf, slot_offset + 0, O7);
 227       __ st_ptr(O7, frame_map()->address_for_monitor_lock(i));
 228       __ ld_ptr(OSR_buf, slot_offset + 1*BytesPerWord, O7);
 229       __ st_ptr(O7, frame_map()->address_for_monitor_object(i));
 230     }
 231   }
 232 }
 233 
 234 
 235 // Optimized Library calls
 236 // This is the fast version of java.lang.String.compare; it has not
 237 // OSR-entry and therefore, we generate a slow version for OSR's
 238 void LIR_Assembler::emit_string_compare(LIR_Opr left, LIR_Opr right, LIR_Opr dst, CodeEmitInfo* info) {
 239   Register str0 = left->as_register();
 240   Register str1 = right->as_register();
 241 
 242   Label Ldone;
 243 
 244   Register result = dst->as_register();
 245   {
 246     // Get a pointer to the first character of string0 in tmp0
 247     //   and get string0.length() in str0
 248     // Get a pointer to the first character of string1 in tmp1
 249     //   and get string1.length() in str1
 250     // Also, get string0.length()-string1.length() in
 251     //   o7 and get the condition code set
 252     // Note: some instructions have been hoisted for better instruction scheduling
 253 
 254     Register tmp0 = L0;
 255     Register tmp1 = L1;
 256     Register tmp2 = L2;
 257 
 258     int  value_offset = java_lang_String:: value_offset_in_bytes(); // char array
 259     if (java_lang_String::has_offset_field()) {
 260       int offset_offset = java_lang_String::offset_offset_in_bytes(); // first character position
 261       int  count_offset = java_lang_String:: count_offset_in_bytes();
 262       __ load_heap_oop(str0, value_offset, tmp0);
 263       __ ld(str0, offset_offset, tmp2);
 264       __ add(tmp0, arrayOopDesc::base_offset_in_bytes(T_CHAR), tmp0);
 265       __ ld(str0, count_offset, str0);
 266       __ sll(tmp2, exact_log2(sizeof(jchar)), tmp2);
 267     } else {
 268       __ load_heap_oop(str0, value_offset, tmp1);
 269       __ add(tmp1, arrayOopDesc::base_offset_in_bytes(T_CHAR), tmp0);
 270       __ ld(tmp1, arrayOopDesc::length_offset_in_bytes(), str0);
 271     }
 272 
 273     // str1 may be null
 274     add_debug_info_for_null_check_here(info);
 275 
 276     if (java_lang_String::has_offset_field()) {
 277       int offset_offset = java_lang_String::offset_offset_in_bytes(); // first character position
 278       int  count_offset = java_lang_String:: count_offset_in_bytes();
 279       __ load_heap_oop(str1, value_offset, tmp1);
 280       __ add(tmp0, tmp2, tmp0);
 281 
 282       __ ld(str1, offset_offset, tmp2);
 283       __ add(tmp1, arrayOopDesc::base_offset_in_bytes(T_CHAR), tmp1);
 284       __ ld(str1, count_offset, str1);
 285       __ sll(tmp2, exact_log2(sizeof(jchar)), tmp2);
 286       __ add(tmp1, tmp2, tmp1);
 287     } else {
 288       __ load_heap_oop(str1, value_offset, tmp2);
 289       __ add(tmp2, arrayOopDesc::base_offset_in_bytes(T_CHAR), tmp1);
 290       __ ld(tmp2, arrayOopDesc::length_offset_in_bytes(), str1);
 291     }
 292     __ subcc(str0, str1, O7);
 293   }
 294 
 295   {
 296     // Compute the minimum of the string lengths, scale it and store it in limit
 297     Register count0 = I0;
 298     Register count1 = I1;
 299     Register limit  = L3;
 300 
 301     Label Lskip;
 302     __ sll(count0, exact_log2(sizeof(jchar)), limit);             // string0 is shorter
 303     __ br(Assembler::greater, true, Assembler::pt, Lskip);
 304     __ delayed()->sll(count1, exact_log2(sizeof(jchar)), limit);  // string1 is shorter
 305     __ bind(Lskip);
 306 
 307     // If either string is empty (or both of them) the result is the difference in lengths
 308     __ cmp(limit, 0);
 309     __ br(Assembler::equal, true, Assembler::pn, Ldone);
 310     __ delayed()->mov(O7, result);  // result is difference in lengths
 311   }
 312 
 313   {
 314     // Neither string is empty
 315     Label Lloop;
 316 
 317     Register base0 = L0;
 318     Register base1 = L1;
 319     Register chr0  = I0;
 320     Register chr1  = I1;
 321     Register limit = L3;
 322 
 323     // Shift base0 and base1 to the end of the arrays, negate limit
 324     __ add(base0, limit, base0);
 325     __ add(base1, limit, base1);
 326     __ neg(limit);  // limit = -min{string0.length(), string1.length()}
 327 
 328     __ lduh(base0, limit, chr0);
 329     __ bind(Lloop);
 330     __ lduh(base1, limit, chr1);
 331     __ subcc(chr0, chr1, chr0);
 332     __ br(Assembler::notZero, false, Assembler::pn, Ldone);
 333     assert(chr0 == result, "result must be pre-placed");
 334     __ delayed()->inccc(limit, sizeof(jchar));
 335     __ br(Assembler::notZero, true, Assembler::pt, Lloop);
 336     __ delayed()->lduh(base0, limit, chr0);
 337   }
 338 
 339   // If strings are equal up to min length, return the length difference.
 340   __ mov(O7, result);
 341 
 342   // Otherwise, return the difference between the first mismatched chars.
 343   __ bind(Ldone);
 344 }
 345 
 346 
 347 // --------------------------------------------------------------------------------------------
 348 
 349 void LIR_Assembler::monitorexit(LIR_Opr obj_opr, LIR_Opr lock_opr, Register hdr, int monitor_no) {
 350   if (!GenerateSynchronizationCode) return;
 351 
 352   Register obj_reg = obj_opr->as_register();
 353   Register lock_reg = lock_opr->as_register();
 354 
 355   Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no);
 356   Register reg = mon_addr.base();
 357   int offset = mon_addr.disp();
 358   // compute pointer to BasicLock
 359   if (mon_addr.is_simm13()) {
 360     __ add(reg, offset, lock_reg);
 361   }
 362   else {
 363     __ set(offset, lock_reg);
 364     __ add(reg, lock_reg, lock_reg);
 365   }
 366   // unlock object
 367   MonitorAccessStub* slow_case = new MonitorExitStub(lock_opr, UseFastLocking, monitor_no);
 368   // _slow_case_stubs->append(slow_case);
 369   // temporary fix: must be created after exceptionhandler, therefore as call stub
 370   _slow_case_stubs->append(slow_case);
 371   if (UseFastLocking) {
 372     // try inlined fast unlocking first, revert to slow locking if it fails
 373     // note: lock_reg points to the displaced header since the displaced header offset is 0!
 374     assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
 375     __ unlock_object(hdr, obj_reg, lock_reg, *slow_case->entry());
 376   } else {
 377     // always do slow unlocking
 378     // note: the slow unlocking code could be inlined here, however if we use
 379     //       slow unlocking, speed doesn't matter anyway and this solution is
 380     //       simpler and requires less duplicated code - additionally, the
 381     //       slow unlocking code is the same in either case which simplifies
 382     //       debugging
 383     __ br(Assembler::always, false, Assembler::pt, *slow_case->entry());
 384     __ delayed()->nop();
 385   }
 386   // done
 387   __ bind(*slow_case->continuation());
 388 }
 389 
 390 
 391 int LIR_Assembler::emit_exception_handler() {
 392   // if the last instruction is a call (typically to do a throw which
 393   // is coming at the end after block reordering) the return address
 394   // must still point into the code area in order to avoid assertion
 395   // failures when searching for the corresponding bci => add a nop
 396   // (was bug 5/14/1999 - gri)
 397   __ nop();
 398 
 399   // generate code for exception handler
 400   ciMethod* method = compilation()->method();
 401 
 402   address handler_base = __ start_a_stub(exception_handler_size);
 403 
 404   if (handler_base == NULL) {
 405     // not enough space left for the handler
 406     bailout("exception handler overflow");
 407     return -1;
 408   }
 409 
 410   int offset = code_offset();
 411 
 412   __ call(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id), relocInfo::runtime_call_type);
 413   __ delayed()->nop();
 414   __ should_not_reach_here();
 415   guarantee(code_offset() - offset <= exception_handler_size, "overflow");
 416   __ end_a_stub();
 417 
 418   return offset;
 419 }
 420 
 421 
 422 // Emit the code to remove the frame from the stack in the exception
 423 // unwind path.
 424 int LIR_Assembler::emit_unwind_handler() {
 425 #ifndef PRODUCT
 426   if (CommentedAssembly) {
 427     _masm->block_comment("Unwind handler");
 428   }
 429 #endif
 430 
 431   int offset = code_offset();
 432 
 433   // Fetch the exception from TLS and clear out exception related thread state
 434   __ ld_ptr(G2_thread, in_bytes(JavaThread::exception_oop_offset()), O0);
 435   __ st_ptr(G0, G2_thread, in_bytes(JavaThread::exception_oop_offset()));
 436   __ st_ptr(G0, G2_thread, in_bytes(JavaThread::exception_pc_offset()));
 437 
 438   __ bind(_unwind_handler_entry);
 439   __ verify_not_null_oop(O0);
 440   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 441     __ mov(O0, I0);  // Preserve the exception
 442   }
 443 
 444   // Preform needed unlocking
 445   MonitorExitStub* stub = NULL;
 446   if (method()->is_synchronized()) {
 447     monitor_address(0, FrameMap::I1_opr);
 448     stub = new MonitorExitStub(FrameMap::I1_opr, true, 0);
 449     __ unlock_object(I3, I2, I1, *stub->entry());
 450     __ bind(*stub->continuation());
 451   }
 452 
 453   if (compilation()->env()->dtrace_method_probes()) {
 454     __ mov(G2_thread, O0);
 455     __ save_thread(I1); // need to preserve thread in G2 across
 456                         // runtime call
 457     metadata2reg(method()->constant_encoding(), O1);
 458     __ call(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), relocInfo::runtime_call_type);
 459     __ delayed()->nop();
 460     __ restore_thread(I1);
 461   }
 462 
 463   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 464     __ mov(I0, O0);  // Restore the exception
 465   }
 466 
 467   // dispatch to the unwind logic
 468   __ call(Runtime1::entry_for(Runtime1::unwind_exception_id), relocInfo::runtime_call_type);
 469   __ delayed()->nop();
 470 
 471   // Emit the slow path assembly
 472   if (stub != NULL) {
 473     stub->emit_code(this);
 474   }
 475 
 476   return offset;
 477 }
 478 
 479 
 480 int LIR_Assembler::emit_deopt_handler() {
 481   // if the last instruction is a call (typically to do a throw which
 482   // is coming at the end after block reordering) the return address
 483   // must still point into the code area in order to avoid assertion
 484   // failures when searching for the corresponding bci => add a nop
 485   // (was bug 5/14/1999 - gri)
 486   __ nop();
 487 
 488   // generate code for deopt handler
 489   ciMethod* method = compilation()->method();
 490   address handler_base = __ start_a_stub(deopt_handler_size);
 491   if (handler_base == NULL) {
 492     // not enough space left for the handler
 493     bailout("deopt handler overflow");
 494     return -1;
 495   }
 496 
 497   int offset = code_offset();
 498   AddressLiteral deopt_blob(SharedRuntime::deopt_blob()->unpack());
 499   __ JUMP(deopt_blob, G3_scratch, 0); // sethi;jmp
 500   __ delayed()->nop();
 501   guarantee(code_offset() - offset <= deopt_handler_size, "overflow");
 502   __ end_a_stub();
 503 
 504   return offset;
 505 }
 506 
 507 
 508 void LIR_Assembler::jobject2reg(jobject o, Register reg) {
 509   if (o == NULL) {
 510     __ set(NULL_WORD, reg);
 511   } else {
 512 #ifdef ASSERT
 513     {
 514       ThreadInVMfromNative tiv(JavaThread::current());
 515       assert(Universe::heap()->is_in_reserved(JNIHandles::resolve(o)), "should be real oop");
 516     }
 517 #endif
 518     int oop_index = __ oop_recorder()->find_index(o);
 519     RelocationHolder rspec = oop_Relocation::spec(oop_index);
 520     __ set(NULL_WORD, reg, rspec); // Will be set when the nmethod is created
 521   }
 522 }
 523 
 524 
 525 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) {
 526   // Allocate a new index in table to hold the object once it's been patched
 527   int oop_index = __ oop_recorder()->allocate_oop_index(NULL);
 528   PatchingStub* patch = new PatchingStub(_masm, patching_id(info), oop_index);
 529 
 530   AddressLiteral addrlit(NULL, oop_Relocation::spec(oop_index));
 531   assert(addrlit.rspec().type() == relocInfo::oop_type, "must be an oop reloc");
 532   // It may not seem necessary to use a sethi/add pair to load a NULL into dest, but the
 533   // NULL will be dynamically patched later and the patched value may be large.  We must
 534   // therefore generate the sethi/add as a placeholders
 535   __ patchable_set(addrlit, reg);
 536 
 537   patching_epilog(patch, lir_patch_normal, reg, info);
 538 }
 539 
 540 
 541 void LIR_Assembler::metadata2reg(Metadata* o, Register reg) {
 542   __ set_metadata_constant(o, reg);
 543 }
 544 
 545 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo *info) {
 546   // Allocate a new index in table to hold the klass once it's been patched
 547   int index = __ oop_recorder()->allocate_metadata_index(NULL);
 548   PatchingStub* patch = new PatchingStub(_masm, PatchingStub::load_klass_id, index);
 549   AddressLiteral addrlit(NULL, metadata_Relocation::spec(index));
 550   assert(addrlit.rspec().type() == relocInfo::metadata_type, "must be an metadata reloc");
 551   // It may not seem necessary to use a sethi/add pair to load a NULL into dest, but the
 552   // NULL will be dynamically patched later and the patched value may be large.  We must
 553   // therefore generate the sethi/add as a placeholders
 554   __ patchable_set(addrlit, reg);
 555 
 556   patching_epilog(patch, lir_patch_normal, reg, info);
 557 }
 558 
 559 void LIR_Assembler::emit_op3(LIR_Op3* op) {
 560   Register Rdividend = op->in_opr1()->as_register();
 561   Register Rdivisor  = noreg;
 562   Register Rscratch  = op->in_opr3()->as_register();
 563   Register Rresult   = op->result_opr()->as_register();
 564   int divisor = -1;
 565 
 566   if (op->in_opr2()->is_register()) {
 567     Rdivisor = op->in_opr2()->as_register();
 568   } else {
 569     divisor = op->in_opr2()->as_constant_ptr()->as_jint();
 570     assert(Assembler::is_simm13(divisor), "can only handle simm13");
 571   }
 572 
 573   assert(Rdividend != Rscratch, "");
 574   assert(Rdivisor  != Rscratch, "");
 575   assert(op->code() == lir_idiv || op->code() == lir_irem, "Must be irem or idiv");
 576 
 577   if (Rdivisor == noreg && is_power_of_2(divisor)) {
 578     // convert division by a power of two into some shifts and logical operations
 579     if (op->code() == lir_idiv) {
 580       if (divisor == 2) {
 581         __ srl(Rdividend, 31, Rscratch);
 582       } else {
 583         __ sra(Rdividend, 31, Rscratch);
 584         __ and3(Rscratch, divisor - 1, Rscratch);
 585       }
 586       __ add(Rdividend, Rscratch, Rscratch);
 587       __ sra(Rscratch, log2_int(divisor), Rresult);
 588       return;
 589     } else {
 590       if (divisor == 2) {
 591         __ srl(Rdividend, 31, Rscratch);
 592       } else {
 593         __ sra(Rdividend, 31, Rscratch);
 594         __ and3(Rscratch, divisor - 1,Rscratch);
 595       }
 596       __ add(Rdividend, Rscratch, Rscratch);
 597       __ andn(Rscratch, divisor - 1,Rscratch);
 598       __ sub(Rdividend, Rscratch, Rresult);
 599       return;
 600     }
 601   }
 602 
 603   __ sra(Rdividend, 31, Rscratch);
 604   __ wry(Rscratch);
 605 
 606   add_debug_info_for_div0_here(op->info());
 607 
 608   if (Rdivisor != noreg) {
 609     __ sdivcc(Rdividend, Rdivisor, (op->code() == lir_idiv ? Rresult : Rscratch));
 610   } else {
 611     assert(Assembler::is_simm13(divisor), "can only handle simm13");
 612     __ sdivcc(Rdividend, divisor, (op->code() == lir_idiv ? Rresult : Rscratch));
 613   }
 614 
 615   Label skip;
 616   __ br(Assembler::overflowSet, true, Assembler::pn, skip);
 617   __ delayed()->Assembler::sethi(0x80000000, (op->code() == lir_idiv ? Rresult : Rscratch));
 618   __ bind(skip);
 619 
 620   if (op->code() == lir_irem) {
 621     if (Rdivisor != noreg) {
 622       __ smul(Rscratch, Rdivisor, Rscratch);
 623     } else {
 624       __ smul(Rscratch, divisor, Rscratch);
 625     }
 626     __ sub(Rdividend, Rscratch, Rresult);
 627   }
 628 }
 629 
 630 
 631 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) {
 632 #ifdef ASSERT
 633   assert(op->block() == NULL || op->block()->label() == op->label(), "wrong label");
 634   if (op->block() != NULL)  _branch_target_blocks.append(op->block());
 635   if (op->ublock() != NULL) _branch_target_blocks.append(op->ublock());
 636 #endif
 637   assert(op->info() == NULL, "shouldn't have CodeEmitInfo");
 638 
 639   if (op->cond() == lir_cond_always) {
 640     __ br(Assembler::always, false, Assembler::pt, *(op->label()));
 641   } else if (op->code() == lir_cond_float_branch) {
 642     assert(op->ublock() != NULL, "must have unordered successor");
 643     bool is_unordered = (op->ublock() == op->block());
 644     Assembler::Condition acond;
 645     switch (op->cond()) {
 646       case lir_cond_equal:         acond = Assembler::f_equal;    break;
 647       case lir_cond_notEqual:      acond = Assembler::f_notEqual; break;
 648       case lir_cond_less:          acond = (is_unordered ? Assembler::f_unorderedOrLess          : Assembler::f_less);           break;
 649       case lir_cond_greater:       acond = (is_unordered ? Assembler::f_unorderedOrGreater       : Assembler::f_greater);        break;
 650       case lir_cond_lessEqual:     acond = (is_unordered ? Assembler::f_unorderedOrLessOrEqual   : Assembler::f_lessOrEqual);    break;
 651       case lir_cond_greaterEqual:  acond = (is_unordered ? Assembler::f_unorderedOrGreaterOrEqual: Assembler::f_greaterOrEqual); break;
 652       default :                         ShouldNotReachHere();
 653     }
 654     __ fb( acond, false, Assembler::pn, *(op->label()));
 655   } else {
 656     assert (op->code() == lir_branch, "just checking");
 657 
 658     Assembler::Condition acond;
 659     switch (op->cond()) {
 660       case lir_cond_equal:        acond = Assembler::equal;                break;
 661       case lir_cond_notEqual:     acond = Assembler::notEqual;             break;
 662       case lir_cond_less:         acond = Assembler::less;                 break;
 663       case lir_cond_lessEqual:    acond = Assembler::lessEqual;            break;
 664       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;         break;
 665       case lir_cond_greater:      acond = Assembler::greater;              break;
 666       case lir_cond_aboveEqual:   acond = Assembler::greaterEqualUnsigned; break;
 667       case lir_cond_belowEqual:   acond = Assembler::lessEqualUnsigned;    break;
 668       default:                         ShouldNotReachHere();
 669     };
 670 
 671     // sparc has different condition codes for testing 32-bit
 672     // vs. 64-bit values.  We could always test xcc is we could
 673     // guarantee that 32-bit loads always sign extended but that isn't
 674     // true and since sign extension isn't free, it would impose a
 675     // slight cost.
 676 #ifdef _LP64
 677     if  (op->type() == T_INT) {
 678       __ br(acond, false, Assembler::pn, *(op->label()));
 679     } else
 680 #endif
 681       __ brx(acond, false, Assembler::pn, *(op->label()));
 682   }
 683   // The peephole pass fills the delay slot
 684 }
 685 
 686 
 687 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) {
 688   Bytecodes::Code code = op->bytecode();
 689   LIR_Opr dst = op->result_opr();
 690 
 691   switch(code) {
 692     case Bytecodes::_i2l: {
 693       Register rlo  = dst->as_register_lo();
 694       Register rhi  = dst->as_register_hi();
 695       Register rval = op->in_opr()->as_register();
 696 #ifdef _LP64
 697       __ sra(rval, 0, rlo);
 698 #else
 699       __ mov(rval, rlo);
 700       __ sra(rval, BitsPerInt-1, rhi);
 701 #endif
 702       break;
 703     }
 704     case Bytecodes::_i2d:
 705     case Bytecodes::_i2f: {
 706       bool is_double = (code == Bytecodes::_i2d);
 707       FloatRegister rdst = is_double ? dst->as_double_reg() : dst->as_float_reg();
 708       FloatRegisterImpl::Width w = is_double ? FloatRegisterImpl::D : FloatRegisterImpl::S;
 709       FloatRegister rsrc = op->in_opr()->as_float_reg();
 710       if (rsrc != rdst) {
 711         __ fmov(FloatRegisterImpl::S, rsrc, rdst);
 712       }
 713       __ fitof(w, rdst, rdst);
 714       break;
 715     }
 716     case Bytecodes::_f2i:{
 717       FloatRegister rsrc = op->in_opr()->as_float_reg();
 718       Address       addr = frame_map()->address_for_slot(dst->single_stack_ix());
 719       Label L;
 720       // result must be 0 if value is NaN; test by comparing value to itself
 721       __ fcmp(FloatRegisterImpl::S, Assembler::fcc0, rsrc, rsrc);
 722       __ fb(Assembler::f_unordered, true, Assembler::pn, L);
 723       __ delayed()->st(G0, addr); // annuled if contents of rsrc is not NaN
 724       __ ftoi(FloatRegisterImpl::S, rsrc, rsrc);
 725       // move integer result from float register to int register
 726       __ stf(FloatRegisterImpl::S, rsrc, addr.base(), addr.disp());
 727       __ bind (L);
 728       break;
 729     }
 730     case Bytecodes::_l2i: {
 731       Register rlo  = op->in_opr()->as_register_lo();
 732       Register rhi  = op->in_opr()->as_register_hi();
 733       Register rdst = dst->as_register();
 734 #ifdef _LP64
 735       __ sra(rlo, 0, rdst);
 736 #else
 737       __ mov(rlo, rdst);
 738 #endif
 739       break;
 740     }
 741     case Bytecodes::_d2f:
 742     case Bytecodes::_f2d: {
 743       bool is_double = (code == Bytecodes::_f2d);
 744       assert((!is_double && dst->is_single_fpu()) || (is_double && dst->is_double_fpu()), "check");
 745       LIR_Opr val = op->in_opr();
 746       FloatRegister rval = (code == Bytecodes::_d2f) ? val->as_double_reg() : val->as_float_reg();
 747       FloatRegister rdst = is_double ? dst->as_double_reg() : dst->as_float_reg();
 748       FloatRegisterImpl::Width vw = is_double ? FloatRegisterImpl::S : FloatRegisterImpl::D;
 749       FloatRegisterImpl::Width dw = is_double ? FloatRegisterImpl::D : FloatRegisterImpl::S;
 750       __ ftof(vw, dw, rval, rdst);
 751       break;
 752     }
 753     case Bytecodes::_i2s:
 754     case Bytecodes::_i2b: {
 755       Register rval = op->in_opr()->as_register();
 756       Register rdst = dst->as_register();
 757       int shift = (code == Bytecodes::_i2b) ? (BitsPerInt - T_BYTE_aelem_bytes * BitsPerByte) : (BitsPerInt - BitsPerShort);
 758       __ sll (rval, shift, rdst);
 759       __ sra (rdst, shift, rdst);
 760       break;
 761     }
 762     case Bytecodes::_i2c: {
 763       Register rval = op->in_opr()->as_register();
 764       Register rdst = dst->as_register();
 765       int shift = BitsPerInt - T_CHAR_aelem_bytes * BitsPerByte;
 766       __ sll (rval, shift, rdst);
 767       __ srl (rdst, shift, rdst);
 768       break;
 769     }
 770 
 771     default: ShouldNotReachHere();
 772   }
 773 }
 774 
 775 
 776 void LIR_Assembler::align_call(LIR_Code) {
 777   // do nothing since all instructions are word aligned on sparc
 778 }
 779 
 780 
 781 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
 782   __ call(op->addr(), rtype);
 783   // The peephole pass fills the delay slot, add_call_info is done in
 784   // LIR_Assembler::emit_delay.
 785 }
 786 
 787 
 788 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
 789   __ ic_call(op->addr(), false);
 790   // The peephole pass fills the delay slot, add_call_info is done in
 791   // LIR_Assembler::emit_delay.
 792 }
 793 
 794 
 795 void LIR_Assembler::vtable_call(LIR_OpJavaCall* op) {
 796   add_debug_info_for_null_check_here(op->info());
 797   __ load_klass(O0, G3_scratch);
 798   if (Assembler::is_simm13(op->vtable_offset())) {
 799     __ ld_ptr(G3_scratch, op->vtable_offset(), G5_method);
 800   } else {
 801     // This will generate 2 instructions
 802     __ set(op->vtable_offset(), G5_method);
 803     // ld_ptr, set_hi, set
 804     __ ld_ptr(G3_scratch, G5_method, G5_method);
 805   }
 806   __ ld_ptr(G5_method, Method::from_compiled_offset(), G3_scratch);
 807   __ callr(G3_scratch, G0);
 808   // the peephole pass fills the delay slot
 809 }
 810 
 811 int LIR_Assembler::store(LIR_Opr from_reg, Register base, int offset, BasicType type, bool wide, bool unaligned) {
 812   int store_offset;
 813   if (!Assembler::is_simm13(offset + (type == T_LONG) ? wordSize : 0)) {
 814     assert(!unaligned, "can't handle this");
 815     // for offsets larger than a simm13 we setup the offset in O7
 816     __ set(offset, O7);
 817     store_offset = store(from_reg, base, O7, type, wide);
 818   } else {
 819     if (type == T_ARRAY || type == T_OBJECT) {
 820       __ verify_oop(from_reg->as_register());
 821     }
 822     store_offset = code_offset();
 823     switch (type) {
 824       case T_BOOLEAN: // fall through
 825       case T_BYTE  : __ stb(from_reg->as_register(), base, offset); break;
 826       case T_CHAR  : __ sth(from_reg->as_register(), base, offset); break;
 827       case T_SHORT : __ sth(from_reg->as_register(), base, offset); break;
 828       case T_INT   : __ stw(from_reg->as_register(), base, offset); break;
 829       case T_LONG  :
 830 #ifdef _LP64
 831         if (unaligned || PatchALot) {
 832           __ srax(from_reg->as_register_lo(), 32, O7);
 833           __ stw(from_reg->as_register_lo(), base, offset + lo_word_offset_in_bytes);
 834           __ stw(O7,                         base, offset + hi_word_offset_in_bytes);
 835         } else {
 836           __ stx(from_reg->as_register_lo(), base, offset);
 837         }
 838 #else
 839         assert(Assembler::is_simm13(offset + 4), "must be");
 840         __ stw(from_reg->as_register_lo(), base, offset + lo_word_offset_in_bytes);
 841         __ stw(from_reg->as_register_hi(), base, offset + hi_word_offset_in_bytes);
 842 #endif
 843         break;
 844       case T_ADDRESS:
 845       case T_METADATA:
 846         __ st_ptr(from_reg->as_register(), base, offset);
 847         break;
 848       case T_ARRAY : // fall through
 849       case T_OBJECT:
 850         {
 851           if (UseCompressedOops && !wide) {
 852             __ encode_heap_oop(from_reg->as_register(), G3_scratch);
 853             store_offset = code_offset();
 854             __ stw(G3_scratch, base, offset);
 855           } else {
 856             __ st_ptr(from_reg->as_register(), base, offset);
 857           }
 858           break;
 859         }
 860 
 861       case T_FLOAT : __ stf(FloatRegisterImpl::S, from_reg->as_float_reg(), base, offset); break;
 862       case T_DOUBLE:
 863         {
 864           FloatRegister reg = from_reg->as_double_reg();
 865           // split unaligned stores
 866           if (unaligned || PatchALot) {
 867             assert(Assembler::is_simm13(offset + 4), "must be");
 868             __ stf(FloatRegisterImpl::S, reg->successor(), base, offset + 4);
 869             __ stf(FloatRegisterImpl::S, reg,              base, offset);
 870           } else {
 871             __ stf(FloatRegisterImpl::D, reg, base, offset);
 872           }
 873           break;
 874         }
 875       default      : ShouldNotReachHere();
 876     }
 877   }
 878   return store_offset;
 879 }
 880 
 881 
 882 int LIR_Assembler::store(LIR_Opr from_reg, Register base, Register disp, BasicType type, bool wide) {
 883   if (type == T_ARRAY || type == T_OBJECT) {
 884     __ verify_oop(from_reg->as_register());
 885   }
 886   int store_offset = code_offset();
 887   switch (type) {
 888     case T_BOOLEAN: // fall through
 889     case T_BYTE  : __ stb(from_reg->as_register(), base, disp); break;
 890     case T_CHAR  : __ sth(from_reg->as_register(), base, disp); break;
 891     case T_SHORT : __ sth(from_reg->as_register(), base, disp); break;
 892     case T_INT   : __ stw(from_reg->as_register(), base, disp); break;
 893     case T_LONG  :
 894 #ifdef _LP64
 895       __ stx(from_reg->as_register_lo(), base, disp);
 896 #else
 897       assert(from_reg->as_register_hi()->successor() == from_reg->as_register_lo(), "must match");
 898       __ std(from_reg->as_register_hi(), base, disp);
 899 #endif
 900       break;
 901     case T_ADDRESS:
 902       __ st_ptr(from_reg->as_register(), base, disp);
 903       break;
 904     case T_ARRAY : // fall through
 905     case T_OBJECT:
 906       {
 907         if (UseCompressedOops && !wide) {
 908           __ encode_heap_oop(from_reg->as_register(), G3_scratch);
 909           store_offset = code_offset();
 910           __ stw(G3_scratch, base, disp);
 911         } else {
 912           __ st_ptr(from_reg->as_register(), base, disp);
 913         }
 914         break;
 915       }
 916     case T_FLOAT : __ stf(FloatRegisterImpl::S, from_reg->as_float_reg(), base, disp); break;
 917     case T_DOUBLE: __ stf(FloatRegisterImpl::D, from_reg->as_double_reg(), base, disp); break;
 918     default      : ShouldNotReachHere();
 919   }
 920   return store_offset;
 921 }
 922 
 923 
 924 int LIR_Assembler::load(Register base, int offset, LIR_Opr to_reg, BasicType type, bool wide, bool unaligned) {
 925   int load_offset;
 926   if (!Assembler::is_simm13(offset + (type == T_LONG) ? wordSize : 0)) {
 927     assert(base != O7, "destroying register");
 928     assert(!unaligned, "can't handle this");
 929     // for offsets larger than a simm13 we setup the offset in O7
 930     __ set(offset, O7);
 931     load_offset = load(base, O7, to_reg, type, wide);
 932   } else {
 933     load_offset = code_offset();
 934     switch(type) {
 935       case T_BOOLEAN: // fall through
 936       case T_BYTE  : __ ldsb(base, offset, to_reg->as_register()); break;
 937       case T_CHAR  : __ lduh(base, offset, to_reg->as_register()); break;
 938       case T_SHORT : __ ldsh(base, offset, to_reg->as_register()); break;
 939       case T_INT   : __ ld(base, offset, to_reg->as_register()); break;
 940       case T_LONG  :
 941         if (!unaligned) {
 942 #ifdef _LP64
 943           __ ldx(base, offset, to_reg->as_register_lo());
 944 #else
 945           assert(to_reg->as_register_hi()->successor() == to_reg->as_register_lo(),
 946                  "must be sequential");
 947           __ ldd(base, offset, to_reg->as_register_hi());
 948 #endif
 949         } else {
 950 #ifdef _LP64
 951           assert(base != to_reg->as_register_lo(), "can't handle this");
 952           assert(O7 != to_reg->as_register_lo(), "can't handle this");
 953           __ ld(base, offset + hi_word_offset_in_bytes, to_reg->as_register_lo());
 954           __ lduw(base, offset + lo_word_offset_in_bytes, O7); // in case O7 is base or offset, use it last
 955           __ sllx(to_reg->as_register_lo(), 32, to_reg->as_register_lo());
 956           __ or3(to_reg->as_register_lo(), O7, to_reg->as_register_lo());
 957 #else
 958           if (base == to_reg->as_register_lo()) {
 959             __ ld(base, offset + hi_word_offset_in_bytes, to_reg->as_register_hi());
 960             __ ld(base, offset + lo_word_offset_in_bytes, to_reg->as_register_lo());
 961           } else {
 962             __ ld(base, offset + lo_word_offset_in_bytes, to_reg->as_register_lo());
 963             __ ld(base, offset + hi_word_offset_in_bytes, to_reg->as_register_hi());
 964           }
 965 #endif
 966         }
 967         break;
 968       case T_METADATA:  __ ld_ptr(base, offset, to_reg->as_register()); break;
 969       case T_ADDRESS:
 970 #ifdef _LP64
 971         if (offset == oopDesc::klass_offset_in_bytes() && UseCompressedClassPointers) {
 972           __ lduw(base, offset, to_reg->as_register());
 973           __ decode_klass_not_null(to_reg->as_register());
 974         } else
 975 #endif
 976         {
 977           __ ld_ptr(base, offset, to_reg->as_register());
 978         }
 979         break;
 980       case T_ARRAY : // fall through
 981       case T_OBJECT:
 982         {
 983           if (UseCompressedOops && !wide) {
 984             __ lduw(base, offset, to_reg->as_register());
 985             __ decode_heap_oop(to_reg->as_register());
 986           } else {
 987             __ ld_ptr(base, offset, to_reg->as_register());
 988           }
 989           break;
 990         }
 991       case T_FLOAT:  __ ldf(FloatRegisterImpl::S, base, offset, to_reg->as_float_reg()); break;
 992       case T_DOUBLE:
 993         {
 994           FloatRegister reg = to_reg->as_double_reg();
 995           // split unaligned loads
 996           if (unaligned || PatchALot) {
 997             __ ldf(FloatRegisterImpl::S, base, offset + 4, reg->successor());
 998             __ ldf(FloatRegisterImpl::S, base, offset,     reg);
 999           } else {
1000             __ ldf(FloatRegisterImpl::D, base, offset, to_reg->as_double_reg());
1001           }
1002           break;
1003         }
1004       default      : ShouldNotReachHere();
1005     }
1006     if (type == T_ARRAY || type == T_OBJECT) {
1007       __ verify_oop(to_reg->as_register());
1008     }
1009   }
1010   return load_offset;
1011 }
1012 
1013 
1014 int LIR_Assembler::load(Register base, Register disp, LIR_Opr to_reg, BasicType type, bool wide) {
1015   int load_offset = code_offset();
1016   switch(type) {
1017     case T_BOOLEAN: // fall through
1018     case T_BYTE  :  __ ldsb(base, disp, to_reg->as_register()); break;
1019     case T_CHAR  :  __ lduh(base, disp, to_reg->as_register()); break;
1020     case T_SHORT :  __ ldsh(base, disp, to_reg->as_register()); break;
1021     case T_INT   :  __ ld(base, disp, to_reg->as_register()); break;
1022     case T_ADDRESS: __ ld_ptr(base, disp, to_reg->as_register()); break;
1023     case T_ARRAY : // fall through
1024     case T_OBJECT:
1025       {
1026           if (UseCompressedOops && !wide) {
1027             __ lduw(base, disp, to_reg->as_register());
1028             __ decode_heap_oop(to_reg->as_register());
1029           } else {
1030             __ ld_ptr(base, disp, to_reg->as_register());
1031           }
1032           break;
1033       }
1034     case T_FLOAT:  __ ldf(FloatRegisterImpl::S, base, disp, to_reg->as_float_reg()); break;
1035     case T_DOUBLE: __ ldf(FloatRegisterImpl::D, base, disp, to_reg->as_double_reg()); break;
1036     case T_LONG  :
1037 #ifdef _LP64
1038       __ ldx(base, disp, to_reg->as_register_lo());
1039 #else
1040       assert(to_reg->as_register_hi()->successor() == to_reg->as_register_lo(),
1041              "must be sequential");
1042       __ ldd(base, disp, to_reg->as_register_hi());
1043 #endif
1044       break;
1045     default      : ShouldNotReachHere();
1046   }
1047   if (type == T_ARRAY || type == T_OBJECT) {
1048     __ verify_oop(to_reg->as_register());
1049   }
1050   return load_offset;
1051 }
1052 
1053 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) {
1054   LIR_Const* c = src->as_constant_ptr();
1055   switch (c->type()) {
1056     case T_INT:
1057     case T_FLOAT: {
1058       Register src_reg = O7;
1059       int value = c->as_jint_bits();
1060       if (value == 0) {
1061         src_reg = G0;
1062       } else {
1063         __ set(value, O7);
1064       }
1065       Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
1066       __ stw(src_reg, addr.base(), addr.disp());
1067       break;
1068     }
1069     case T_ADDRESS: {
1070       Register src_reg = O7;
1071       int value = c->as_jint_bits();
1072       if (value == 0) {
1073         src_reg = G0;
1074       } else {
1075         __ set(value, O7);
1076       }
1077       Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
1078       __ st_ptr(src_reg, addr.base(), addr.disp());
1079       break;
1080     }
1081     case T_OBJECT: {
1082       Register src_reg = O7;
1083       jobject2reg(c->as_jobject(), src_reg);
1084       Address addr = frame_map()->address_for_slot(dest->single_stack_ix());
1085       __ st_ptr(src_reg, addr.base(), addr.disp());
1086       break;
1087     }
1088     case T_LONG:
1089     case T_DOUBLE: {
1090       Address addr = frame_map()->address_for_double_slot(dest->double_stack_ix());
1091 
1092       Register tmp = O7;
1093       int value_lo = c->as_jint_lo_bits();
1094       if (value_lo == 0) {
1095         tmp = G0;
1096       } else {
1097         __ set(value_lo, O7);
1098       }
1099       __ stw(tmp, addr.base(), addr.disp() + lo_word_offset_in_bytes);
1100       int value_hi = c->as_jint_hi_bits();
1101       if (value_hi == 0) {
1102         tmp = G0;
1103       } else {
1104         __ set(value_hi, O7);
1105       }
1106       __ stw(tmp, addr.base(), addr.disp() + hi_word_offset_in_bytes);
1107       break;
1108     }
1109     default:
1110       Unimplemented();
1111   }
1112 }
1113 
1114 
1115 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) {
1116   LIR_Const* c = src->as_constant_ptr();
1117   LIR_Address* addr     = dest->as_address_ptr();
1118   Register base = addr->base()->as_pointer_register();
1119   int offset = -1;
1120 
1121   switch (c->type()) {
1122     case T_INT:
1123     case T_FLOAT:
1124     case T_ADDRESS: {
1125       LIR_Opr tmp = FrameMap::O7_opr;
1126       int value = c->as_jint_bits();
1127       if (value == 0) {
1128         tmp = FrameMap::G0_opr;
1129       } else if (Assembler::is_simm13(value)) {
1130         __ set(value, O7);
1131       }
1132       if (addr->index()->is_valid()) {
1133         assert(addr->disp() == 0, "must be zero");
1134         offset = store(tmp, base, addr->index()->as_pointer_register(), type, wide);
1135       } else {
1136         assert(Assembler::is_simm13(addr->disp()), "can't handle larger addresses");
1137         offset = store(tmp, base, addr->disp(), type, wide, false);
1138       }
1139       break;
1140     }
1141     case T_LONG:
1142     case T_DOUBLE: {
1143       assert(!addr->index()->is_valid(), "can't handle reg reg address here");
1144       assert(Assembler::is_simm13(addr->disp()) &&
1145              Assembler::is_simm13(addr->disp() + 4), "can't handle larger addresses");
1146 
1147       LIR_Opr tmp = FrameMap::O7_opr;
1148       int value_lo = c->as_jint_lo_bits();
1149       if (value_lo == 0) {
1150         tmp = FrameMap::G0_opr;
1151       } else {
1152         __ set(value_lo, O7);
1153       }
1154       offset = store(tmp, base, addr->disp() + lo_word_offset_in_bytes, T_INT, wide, false);
1155       int value_hi = c->as_jint_hi_bits();
1156       if (value_hi == 0) {
1157         tmp = FrameMap::G0_opr;
1158       } else {
1159         __ set(value_hi, O7);
1160       }
1161       store(tmp, base, addr->disp() + hi_word_offset_in_bytes, T_INT, wide, false);
1162       break;
1163     }
1164     case T_OBJECT: {
1165       jobject obj = c->as_jobject();
1166       LIR_Opr tmp;
1167       if (obj == NULL) {
1168         tmp = FrameMap::G0_opr;
1169       } else {
1170         tmp = FrameMap::O7_opr;
1171         jobject2reg(c->as_jobject(), O7);
1172       }
1173       // handle either reg+reg or reg+disp address
1174       if (addr->index()->is_valid()) {
1175         assert(addr->disp() == 0, "must be zero");
1176         offset = store(tmp, base, addr->index()->as_pointer_register(), type, wide);
1177       } else {
1178         assert(Assembler::is_simm13(addr->disp()), "can't handle larger addresses");
1179         offset = store(tmp, base, addr->disp(), type, wide, false);
1180       }
1181 
1182       break;
1183     }
1184     default:
1185       Unimplemented();
1186   }
1187   if (info != NULL) {
1188     assert(offset != -1, "offset should've been set");
1189     add_debug_info_for_null_check(offset, info);
1190   }
1191 }
1192 
1193 
1194 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) {
1195   LIR_Const* c = src->as_constant_ptr();
1196   LIR_Opr to_reg = dest;
1197 
1198   switch (c->type()) {
1199     case T_INT:
1200     case T_ADDRESS:
1201       {
1202         jint con = c->as_jint();
1203         if (to_reg->is_single_cpu()) {
1204           assert(patch_code == lir_patch_none, "no patching handled here");
1205           __ set(con, to_reg->as_register());
1206         } else {
1207           ShouldNotReachHere();
1208           assert(to_reg->is_single_fpu(), "wrong register kind");
1209 
1210           __ set(con, O7);
1211           Address temp_slot(SP, (frame::register_save_words * wordSize) + STACK_BIAS);
1212           __ st(O7, temp_slot);
1213           __ ldf(FloatRegisterImpl::S, temp_slot, to_reg->as_float_reg());
1214         }
1215       }
1216       break;
1217 
1218     case T_LONG:
1219       {
1220         jlong con = c->as_jlong();
1221 
1222         if (to_reg->is_double_cpu()) {
1223 #ifdef _LP64
1224           __ set(con,  to_reg->as_register_lo());
1225 #else
1226           __ set(low(con),  to_reg->as_register_lo());
1227           __ set(high(con), to_reg->as_register_hi());
1228 #endif
1229 #ifdef _LP64
1230         } else if (to_reg->is_single_cpu()) {
1231           __ set(con, to_reg->as_register());
1232 #endif
1233         } else {
1234           ShouldNotReachHere();
1235           assert(to_reg->is_double_fpu(), "wrong register kind");
1236           Address temp_slot_lo(SP, ((frame::register_save_words  ) * wordSize) + STACK_BIAS);
1237           Address temp_slot_hi(SP, ((frame::register_save_words) * wordSize) + (longSize/2) + STACK_BIAS);
1238           __ set(low(con),  O7);
1239           __ st(O7, temp_slot_lo);
1240           __ set(high(con), O7);
1241           __ st(O7, temp_slot_hi);
1242           __ ldf(FloatRegisterImpl::D, temp_slot_lo, to_reg->as_double_reg());
1243         }
1244       }
1245       break;
1246 
1247     case T_OBJECT:
1248       {
1249         if (patch_code == lir_patch_none) {
1250           jobject2reg(c->as_jobject(), to_reg->as_register());
1251         } else {
1252           jobject2reg_with_patching(to_reg->as_register(), info);
1253         }
1254       }
1255       break;
1256 
1257     case T_METADATA:
1258       {
1259         if (patch_code == lir_patch_none) {
1260           metadata2reg(c->as_metadata(), to_reg->as_register());
1261         } else {
1262           klass2reg_with_patching(to_reg->as_register(), info);
1263         }
1264       }
1265       break;
1266 
1267     case T_FLOAT:
1268       {
1269         address const_addr = __ float_constant(c->as_jfloat());
1270         if (const_addr == NULL) {
1271           bailout("const section overflow");
1272           break;
1273         }
1274         RelocationHolder rspec = internal_word_Relocation::spec(const_addr);
1275         AddressLiteral const_addrlit(const_addr, rspec);
1276         if (to_reg->is_single_fpu()) {
1277           __ patchable_sethi(const_addrlit, O7);
1278           __ relocate(rspec);
1279           __ ldf(FloatRegisterImpl::S, O7, const_addrlit.low10(), to_reg->as_float_reg());
1280 
1281         } else {
1282           assert(to_reg->is_single_cpu(), "Must be a cpu register.");
1283 
1284           __ set(const_addrlit, O7);
1285           __ ld(O7, 0, to_reg->as_register());
1286         }
1287       }
1288       break;
1289 
1290     case T_DOUBLE:
1291       {
1292         address const_addr = __ double_constant(c->as_jdouble());
1293         if (const_addr == NULL) {
1294           bailout("const section overflow");
1295           break;
1296         }
1297         RelocationHolder rspec = internal_word_Relocation::spec(const_addr);
1298 
1299         if (to_reg->is_double_fpu()) {
1300           AddressLiteral const_addrlit(const_addr, rspec);
1301           __ patchable_sethi(const_addrlit, O7);
1302           __ relocate(rspec);
1303           __ ldf (FloatRegisterImpl::D, O7, const_addrlit.low10(), to_reg->as_double_reg());
1304         } else {
1305           assert(to_reg->is_double_cpu(), "Must be a long register.");
1306 #ifdef _LP64
1307           __ set(jlong_cast(c->as_jdouble()), to_reg->as_register_lo());
1308 #else
1309           __ set(low(jlong_cast(c->as_jdouble())), to_reg->as_register_lo());
1310           __ set(high(jlong_cast(c->as_jdouble())), to_reg->as_register_hi());
1311 #endif
1312         }
1313 
1314       }
1315       break;
1316 
1317     default:
1318       ShouldNotReachHere();
1319   }
1320 }
1321 
1322 Address LIR_Assembler::as_Address(LIR_Address* addr) {
1323   Register reg = addr->base()->as_pointer_register();
1324   LIR_Opr index = addr->index();
1325   if (index->is_illegal()) {
1326     return Address(reg, addr->disp());
1327   } else {
1328     assert (addr->disp() == 0, "unsupported address mode");
1329     return Address(reg, index->as_pointer_register());
1330   }
1331 }
1332 
1333 
1334 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) {
1335   switch (type) {
1336     case T_INT:
1337     case T_FLOAT: {
1338       Register tmp = O7;
1339       Address from = frame_map()->address_for_slot(src->single_stack_ix());
1340       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
1341       __ lduw(from.base(), from.disp(), tmp);
1342       __ stw(tmp, to.base(), to.disp());
1343       break;
1344     }
1345     case T_OBJECT: {
1346       Register tmp = O7;
1347       Address from = frame_map()->address_for_slot(src->single_stack_ix());
1348       Address to   = frame_map()->address_for_slot(dest->single_stack_ix());
1349       __ ld_ptr(from.base(), from.disp(), tmp);
1350       __ st_ptr(tmp, to.base(), to.disp());
1351       break;
1352     }
1353     case T_LONG:
1354     case T_DOUBLE: {
1355       Register tmp = O7;
1356       Address from = frame_map()->address_for_double_slot(src->double_stack_ix());
1357       Address to   = frame_map()->address_for_double_slot(dest->double_stack_ix());
1358       __ lduw(from.base(), from.disp(), tmp);
1359       __ stw(tmp, to.base(), to.disp());
1360       __ lduw(from.base(), from.disp() + 4, tmp);
1361       __ stw(tmp, to.base(), to.disp() + 4);
1362       break;
1363     }
1364 
1365     default:
1366       ShouldNotReachHere();
1367   }
1368 }
1369 
1370 
1371 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) {
1372   Address base = as_Address(addr);
1373   return Address(base.base(), base.disp() + hi_word_offset_in_bytes);
1374 }
1375 
1376 
1377 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) {
1378   Address base = as_Address(addr);
1379   return Address(base.base(), base.disp() + lo_word_offset_in_bytes);
1380 }
1381 
1382 
1383 void LIR_Assembler::mem2reg(LIR_Opr src_opr, LIR_Opr dest, BasicType type,
1384                             LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide, bool unaligned) {
1385 
1386   assert(type != T_METADATA, "load of metadata ptr not supported");
1387   LIR_Address* addr = src_opr->as_address_ptr();
1388   LIR_Opr to_reg = dest;
1389 
1390   Register src = addr->base()->as_pointer_register();
1391   Register disp_reg = noreg;
1392   int disp_value = addr->disp();
1393   bool needs_patching = (patch_code != lir_patch_none);
1394 
1395   if (addr->base()->type() == T_OBJECT) {
1396     __ verify_oop(src);
1397   }
1398 
1399   PatchingStub* patch = NULL;
1400   if (needs_patching) {
1401     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1402     assert(!to_reg->is_double_cpu() ||
1403            patch_code == lir_patch_none ||
1404            patch_code == lir_patch_normal, "patching doesn't match register");
1405   }
1406 
1407   if (addr->index()->is_illegal()) {
1408     if (!Assembler::is_simm13(disp_value) && (!unaligned || Assembler::is_simm13(disp_value + 4))) {
1409       if (needs_patching) {
1410         __ patchable_set(0, O7);
1411       } else {
1412         __ set(disp_value, O7);
1413       }
1414       disp_reg = O7;
1415     }
1416   } else if (unaligned || PatchALot) {
1417     __ add(src, addr->index()->as_register(), O7);
1418     src = O7;
1419   } else {
1420     disp_reg = addr->index()->as_pointer_register();
1421     assert(disp_value == 0, "can't handle 3 operand addresses");
1422   }
1423 
1424   // remember the offset of the load.  The patching_epilog must be done
1425   // before the call to add_debug_info, otherwise the PcDescs don't get
1426   // entered in increasing order.
1427   int offset = code_offset();
1428 
1429   assert(disp_reg != noreg || Assembler::is_simm13(disp_value), "should have set this up");
1430   if (disp_reg == noreg) {
1431     offset = load(src, disp_value, to_reg, type, wide, unaligned);
1432   } else {
1433     assert(!unaligned, "can't handle this");
1434     offset = load(src, disp_reg, to_reg, type, wide);
1435   }
1436 
1437   if (patch != NULL) {
1438     patching_epilog(patch, patch_code, src, info);
1439   }
1440   if (info != NULL) add_debug_info_for_null_check(offset, info);
1441 }
1442 
1443 
1444 void LIR_Assembler::prefetchr(LIR_Opr src) {
1445   LIR_Address* addr = src->as_address_ptr();
1446   Address from_addr = as_Address(addr);
1447 
1448   if (VM_Version::has_v9()) {
1449     __ prefetch(from_addr, Assembler::severalReads);
1450   }
1451 }
1452 
1453 
1454 void LIR_Assembler::prefetchw(LIR_Opr src) {
1455   LIR_Address* addr = src->as_address_ptr();
1456   Address from_addr = as_Address(addr);
1457 
1458   if (VM_Version::has_v9()) {
1459     __ prefetch(from_addr, Assembler::severalWritesAndPossiblyReads);
1460   }
1461 }
1462 
1463 
1464 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) {
1465   Address addr;
1466   if (src->is_single_word()) {
1467     addr = frame_map()->address_for_slot(src->single_stack_ix());
1468   } else if (src->is_double_word())  {
1469     addr = frame_map()->address_for_double_slot(src->double_stack_ix());
1470   }
1471 
1472   bool unaligned = (addr.disp() - STACK_BIAS) % 8 != 0;
1473   load(addr.base(), addr.disp(), dest, dest->type(), true /*wide*/, unaligned);
1474 }
1475 
1476 
1477 void LIR_Assembler::reg2stack(LIR_Opr from_reg, LIR_Opr dest, BasicType type, bool pop_fpu_stack) {
1478   Address addr;
1479   if (dest->is_single_word()) {
1480     addr = frame_map()->address_for_slot(dest->single_stack_ix());
1481   } else if (dest->is_double_word())  {
1482     addr = frame_map()->address_for_slot(dest->double_stack_ix());
1483   }
1484   bool unaligned = (addr.disp() - STACK_BIAS) % 8 != 0;
1485   store(from_reg, addr.base(), addr.disp(), from_reg->type(), true /*wide*/, unaligned);
1486 }
1487 
1488 
1489 void LIR_Assembler::reg2reg(LIR_Opr from_reg, LIR_Opr to_reg) {
1490   if (from_reg->is_float_kind() && to_reg->is_float_kind()) {
1491     if (from_reg->is_double_fpu()) {
1492       // double to double moves
1493       assert(to_reg->is_double_fpu(), "should match");
1494       __ fmov(FloatRegisterImpl::D, from_reg->as_double_reg(), to_reg->as_double_reg());
1495     } else {
1496       // float to float moves
1497       assert(to_reg->is_single_fpu(), "should match");
1498       __ fmov(FloatRegisterImpl::S, from_reg->as_float_reg(), to_reg->as_float_reg());
1499     }
1500   } else if (!from_reg->is_float_kind() && !to_reg->is_float_kind()) {
1501     if (from_reg->is_double_cpu()) {
1502 #ifdef _LP64
1503       __ mov(from_reg->as_pointer_register(), to_reg->as_pointer_register());
1504 #else
1505       assert(to_reg->is_double_cpu() &&
1506              from_reg->as_register_hi() != to_reg->as_register_lo() &&
1507              from_reg->as_register_lo() != to_reg->as_register_hi(),
1508              "should both be long and not overlap");
1509       // long to long moves
1510       __ mov(from_reg->as_register_hi(), to_reg->as_register_hi());
1511       __ mov(from_reg->as_register_lo(), to_reg->as_register_lo());
1512 #endif
1513 #ifdef _LP64
1514     } else if (to_reg->is_double_cpu()) {
1515       // int to int moves
1516       __ mov(from_reg->as_register(), to_reg->as_register_lo());
1517 #endif
1518     } else {
1519       // int to int moves
1520       __ mov(from_reg->as_register(), to_reg->as_register());
1521     }
1522   } else {
1523     ShouldNotReachHere();
1524   }
1525   if (to_reg->type() == T_OBJECT || to_reg->type() == T_ARRAY) {
1526     __ verify_oop(to_reg->as_register());
1527   }
1528 }
1529 
1530 
1531 void LIR_Assembler::reg2mem(LIR_Opr from_reg, LIR_Opr dest, BasicType type,
1532                             LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack,
1533                             bool wide, bool unaligned) {
1534   assert(type != T_METADATA, "store of metadata ptr not supported");
1535   LIR_Address* addr = dest->as_address_ptr();
1536 
1537   Register src = addr->base()->as_pointer_register();
1538   Register disp_reg = noreg;
1539   int disp_value = addr->disp();
1540   bool needs_patching = (patch_code != lir_patch_none);
1541 
1542   if (addr->base()->is_oop_register()) {
1543     __ verify_oop(src);
1544   }
1545 
1546   PatchingStub* patch = NULL;
1547   if (needs_patching) {
1548     patch = new PatchingStub(_masm, PatchingStub::access_field_id);
1549     assert(!from_reg->is_double_cpu() ||
1550            patch_code == lir_patch_none ||
1551            patch_code == lir_patch_normal, "patching doesn't match register");
1552   }
1553 
1554   if (addr->index()->is_illegal()) {
1555     if (!Assembler::is_simm13(disp_value) && (!unaligned || Assembler::is_simm13(disp_value + 4))) {
1556       if (needs_patching) {
1557         __ patchable_set(0, O7);
1558       } else {
1559         __ set(disp_value, O7);
1560       }
1561       disp_reg = O7;
1562     }
1563   } else if (unaligned || PatchALot) {
1564     __ add(src, addr->index()->as_register(), O7);
1565     src = O7;
1566   } else {
1567     disp_reg = addr->index()->as_pointer_register();
1568     assert(disp_value == 0, "can't handle 3 operand addresses");
1569   }
1570 
1571   // remember the offset of the store.  The patching_epilog must be done
1572   // before the call to add_debug_info_for_null_check, otherwise the PcDescs don't get
1573   // entered in increasing order.
1574   int offset;
1575 
1576   assert(disp_reg != noreg || Assembler::is_simm13(disp_value), "should have set this up");
1577   if (disp_reg == noreg) {
1578     offset = store(from_reg, src, disp_value, type, wide, unaligned);
1579   } else {
1580     assert(!unaligned, "can't handle this");
1581     offset = store(from_reg, src, disp_reg, type, wide);
1582   }
1583 
1584   if (patch != NULL) {
1585     patching_epilog(patch, patch_code, src, info);
1586   }
1587 
1588   if (info != NULL) add_debug_info_for_null_check(offset, info);
1589 }
1590 
1591 
1592 void LIR_Assembler::return_op(LIR_Opr result) {
1593   // the poll may need a register so just pick one that isn't the return register
1594 #if defined(TIERED) && !defined(_LP64)
1595   if (result->type_field() == LIR_OprDesc::long_type) {
1596     // Must move the result to G1
1597     // Must leave proper result in O0,O1 and G1 (TIERED only)
1598     __ sllx(I0, 32, G1);          // Shift bits into high G1
1599     __ srl (I1, 0, I1);           // Zero extend O1 (harmless?)
1600     __ or3 (I1, G1, G1);          // OR 64 bits into G1
1601 #ifdef ASSERT
1602     // mangle it so any problems will show up
1603     __ set(0xdeadbeef, I0);
1604     __ set(0xdeadbeef, I1);
1605 #endif
1606   }
1607 #endif // TIERED
1608   __ set((intptr_t)os::get_polling_page(), L0);
1609   __ relocate(relocInfo::poll_return_type);
1610   __ ld_ptr(L0, 0, G0);
1611   __ ret();
1612   __ delayed()->restore();
1613 }
1614 
1615 
1616 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
1617   __ set((intptr_t)os::get_polling_page(), tmp->as_register());
1618   if (info != NULL) {
1619     add_debug_info_for_branch(info);
1620   } else {
1621     __ relocate(relocInfo::poll_type);
1622   }
1623 
1624   int offset = __ offset();
1625   __ ld_ptr(tmp->as_register(), 0, G0);
1626 
1627   return offset;
1628 }
1629 
1630 
1631 void LIR_Assembler::emit_static_call_stub() {
1632   address call_pc = __ pc();
1633   address stub = __ start_a_stub(call_stub_size);
1634   if (stub == NULL) {
1635     bailout("static call stub overflow");
1636     return;
1637   }
1638 
1639   int start = __ offset();
1640   __ relocate(static_stub_Relocation::spec(call_pc));
1641 
1642   __ set_metadata(NULL, G5);
1643   // must be set to -1 at code generation time
1644   AddressLiteral addrlit(-1);
1645   __ jump_to(addrlit, G3);
1646   __ delayed()->nop();
1647 
1648   assert(__ offset() - start <= call_stub_size, "stub too big");
1649   __ end_a_stub();
1650 }
1651 
1652 
1653 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Op2* op) {
1654   if (opr1->is_single_fpu()) {
1655     __ fcmp(FloatRegisterImpl::S, Assembler::fcc0, opr1->as_float_reg(), opr2->as_float_reg());
1656   } else if (opr1->is_double_fpu()) {
1657     __ fcmp(FloatRegisterImpl::D, Assembler::fcc0, opr1->as_double_reg(), opr2->as_double_reg());
1658   } else if (opr1->is_single_cpu()) {
1659     if (opr2->is_constant()) {
1660       switch (opr2->as_constant_ptr()->type()) {
1661         case T_INT:
1662           { jint con = opr2->as_constant_ptr()->as_jint();
1663             if (Assembler::is_simm13(con)) {
1664               __ cmp(opr1->as_register(), con);
1665             } else {
1666               __ set(con, O7);
1667               __ cmp(opr1->as_register(), O7);
1668             }
1669           }
1670           break;
1671 
1672         case T_OBJECT:
1673           // there are only equal/notequal comparisions on objects
1674           { jobject con = opr2->as_constant_ptr()->as_jobject();
1675             if (con == NULL) {
1676               __ cmp(opr1->as_register(), 0);
1677             } else {
1678               jobject2reg(con, O7);
1679               __ cmp(opr1->as_register(), O7);
1680             }
1681           }
1682           break;
1683 
1684         default:
1685           ShouldNotReachHere();
1686           break;
1687       }
1688     } else {
1689       if (opr2->is_address()) {
1690         LIR_Address * addr = opr2->as_address_ptr();
1691         BasicType type = addr->type();
1692         if ( type == T_OBJECT ) __ ld_ptr(as_Address(addr), O7);
1693         else                    __ ld(as_Address(addr), O7);
1694         __ cmp(opr1->as_register(), O7);
1695       } else {
1696         __ cmp(opr1->as_register(), opr2->as_register());
1697       }
1698     }
1699   } else if (opr1->is_double_cpu()) {
1700     Register xlo = opr1->as_register_lo();
1701     Register xhi = opr1->as_register_hi();
1702     if (opr2->is_constant() && opr2->as_jlong() == 0) {
1703       assert(condition == lir_cond_equal || condition == lir_cond_notEqual, "only handles these cases");
1704 #ifdef _LP64
1705       __ orcc(xhi, G0, G0);
1706 #else
1707       __ orcc(xhi, xlo, G0);
1708 #endif
1709     } else if (opr2->is_register()) {
1710       Register ylo = opr2->as_register_lo();
1711       Register yhi = opr2->as_register_hi();
1712 #ifdef _LP64
1713       __ cmp(xlo, ylo);
1714 #else
1715       __ subcc(xlo, ylo, xlo);
1716       __ subccc(xhi, yhi, xhi);
1717       if (condition == lir_cond_equal || condition == lir_cond_notEqual) {
1718         __ orcc(xhi, xlo, G0);
1719       }
1720 #endif
1721     } else {
1722       ShouldNotReachHere();
1723     }
1724   } else if (opr1->is_address()) {
1725     LIR_Address * addr = opr1->as_address_ptr();
1726     BasicType type = addr->type();
1727     assert (opr2->is_constant(), "Checking");
1728     if ( type == T_OBJECT ) __ ld_ptr(as_Address(addr), O7);
1729     else                    __ ld(as_Address(addr), O7);
1730     __ cmp(O7, opr2->as_constant_ptr()->as_jint());
1731   } else {
1732     ShouldNotReachHere();
1733   }
1734 }
1735 
1736 
1737 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op){
1738   if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) {
1739     bool is_unordered_less = (code == lir_ucmp_fd2i);
1740     if (left->is_single_fpu()) {
1741       __ float_cmp(true, is_unordered_less ? -1 : 1, left->as_float_reg(), right->as_float_reg(), dst->as_register());
1742     } else if (left->is_double_fpu()) {
1743       __ float_cmp(false, is_unordered_less ? -1 : 1, left->as_double_reg(), right->as_double_reg(), dst->as_register());
1744     } else {
1745       ShouldNotReachHere();
1746     }
1747   } else if (code == lir_cmp_l2i) {
1748 #ifdef _LP64
1749     __ lcmp(left->as_register_lo(), right->as_register_lo(), dst->as_register());
1750 #else
1751     __ lcmp(left->as_register_hi(),  left->as_register_lo(),
1752             right->as_register_hi(), right->as_register_lo(),
1753             dst->as_register());
1754 #endif
1755   } else {
1756     ShouldNotReachHere();
1757   }
1758 }
1759 
1760 
1761 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type) {
1762   Assembler::Condition acond;
1763   switch (condition) {
1764     case lir_cond_equal:        acond = Assembler::equal;        break;
1765     case lir_cond_notEqual:     acond = Assembler::notEqual;     break;
1766     case lir_cond_less:         acond = Assembler::less;         break;
1767     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    break;
1768     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; break;
1769     case lir_cond_greater:      acond = Assembler::greater;      break;
1770     case lir_cond_aboveEqual:   acond = Assembler::greaterEqualUnsigned;      break;
1771     case lir_cond_belowEqual:   acond = Assembler::lessEqualUnsigned;      break;
1772     default:                         ShouldNotReachHere();
1773   };
1774 
1775   if (opr1->is_constant() && opr1->type() == T_INT) {
1776     Register dest = result->as_register();
1777     // load up first part of constant before branch
1778     // and do the rest in the delay slot.
1779     if (!Assembler::is_simm13(opr1->as_jint())) {
1780       __ sethi(opr1->as_jint(), dest);
1781     }
1782   } else if (opr1->is_constant()) {
1783     const2reg(opr1, result, lir_patch_none, NULL);
1784   } else if (opr1->is_register()) {
1785     reg2reg(opr1, result);
1786   } else if (opr1->is_stack()) {
1787     stack2reg(opr1, result, result->type());
1788   } else {
1789     ShouldNotReachHere();
1790   }
1791   Label skip;
1792 #ifdef _LP64
1793     if  (type == T_INT) {
1794       __ br(acond, false, Assembler::pt, skip);
1795     } else
1796 #endif
1797       __ brx(acond, false, Assembler::pt, skip); // checks icc on 32bit and xcc on 64bit
1798   if (opr1->is_constant() && opr1->type() == T_INT) {
1799     Register dest = result->as_register();
1800     if (Assembler::is_simm13(opr1->as_jint())) {
1801       __ delayed()->or3(G0, opr1->as_jint(), dest);
1802     } else {
1803       // the sethi has been done above, so just put in the low 10 bits
1804       __ delayed()->or3(dest, opr1->as_jint() & 0x3ff, dest);
1805     }
1806   } else {
1807     // can't do anything useful in the delay slot
1808     __ delayed()->nop();
1809   }
1810   if (opr2->is_constant()) {
1811     const2reg(opr2, result, lir_patch_none, NULL);
1812   } else if (opr2->is_register()) {
1813     reg2reg(opr2, result);
1814   } else if (opr2->is_stack()) {
1815     stack2reg(opr2, result, result->type());
1816   } else {
1817     ShouldNotReachHere();
1818   }
1819   __ bind(skip);
1820 }
1821 
1822 
1823 void LIR_Assembler::arith_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest, CodeEmitInfo* info, bool pop_fpu_stack) {
1824   assert(info == NULL, "unused on this code path");
1825   assert(left->is_register(), "wrong items state");
1826   assert(dest->is_register(), "wrong items state");
1827 
1828   if (right->is_register()) {
1829     if (dest->is_float_kind()) {
1830 
1831       FloatRegister lreg, rreg, res;
1832       FloatRegisterImpl::Width w;
1833       if (right->is_single_fpu()) {
1834         w = FloatRegisterImpl::S;
1835         lreg = left->as_float_reg();
1836         rreg = right->as_float_reg();
1837         res  = dest->as_float_reg();
1838       } else {
1839         w = FloatRegisterImpl::D;
1840         lreg = left->as_double_reg();
1841         rreg = right->as_double_reg();
1842         res  = dest->as_double_reg();
1843       }
1844 
1845       switch (code) {
1846         case lir_add: __ fadd(w, lreg, rreg, res); break;
1847         case lir_sub: __ fsub(w, lreg, rreg, res); break;
1848         case lir_mul: // fall through
1849         case lir_mul_strictfp: __ fmul(w, lreg, rreg, res); break;
1850         case lir_div: // fall through
1851         case lir_div_strictfp: __ fdiv(w, lreg, rreg, res); break;
1852         default: ShouldNotReachHere();
1853       }
1854 
1855     } else if (dest->is_double_cpu()) {
1856 #ifdef _LP64
1857       Register dst_lo = dest->as_register_lo();
1858       Register op1_lo = left->as_pointer_register();
1859       Register op2_lo = right->as_pointer_register();
1860 
1861       switch (code) {
1862         case lir_add:
1863           __ add(op1_lo, op2_lo, dst_lo);
1864           break;
1865 
1866         case lir_sub:
1867           __ sub(op1_lo, op2_lo, dst_lo);
1868           break;
1869 
1870         default: ShouldNotReachHere();
1871       }
1872 #else
1873       Register op1_lo = left->as_register_lo();
1874       Register op1_hi = left->as_register_hi();
1875       Register op2_lo = right->as_register_lo();
1876       Register op2_hi = right->as_register_hi();
1877       Register dst_lo = dest->as_register_lo();
1878       Register dst_hi = dest->as_register_hi();
1879 
1880       switch (code) {
1881         case lir_add:
1882           __ addcc(op1_lo, op2_lo, dst_lo);
1883           __ addc (op1_hi, op2_hi, dst_hi);
1884           break;
1885 
1886         case lir_sub:
1887           __ subcc(op1_lo, op2_lo, dst_lo);
1888           __ subc (op1_hi, op2_hi, dst_hi);
1889           break;
1890 
1891         default: ShouldNotReachHere();
1892       }
1893 #endif
1894     } else {
1895       assert (right->is_single_cpu(), "Just Checking");
1896 
1897       Register lreg = left->as_register();
1898       Register res  = dest->as_register();
1899       Register rreg = right->as_register();
1900       switch (code) {
1901         case lir_add:  __ add  (lreg, rreg, res); break;
1902         case lir_sub:  __ sub  (lreg, rreg, res); break;
1903         case lir_mul:  __ mulx (lreg, rreg, res); break;
1904         default: ShouldNotReachHere();
1905       }
1906     }
1907   } else {
1908     assert (right->is_constant(), "must be constant");
1909 
1910     if (dest->is_single_cpu()) {
1911       Register lreg = left->as_register();
1912       Register res  = dest->as_register();
1913       int    simm13 = right->as_constant_ptr()->as_jint();
1914 
1915       switch (code) {
1916         case lir_add:  __ add  (lreg, simm13, res); break;
1917         case lir_sub:  __ sub  (lreg, simm13, res); break;
1918         case lir_mul:  __ mulx (lreg, simm13, res); break;
1919         default: ShouldNotReachHere();
1920       }
1921     } else {
1922       Register lreg = left->as_pointer_register();
1923       Register res  = dest->as_register_lo();
1924       long con = right->as_constant_ptr()->as_jlong();
1925       assert(Assembler::is_simm13(con), "must be simm13");
1926 
1927       switch (code) {
1928         case lir_add:  __ add  (lreg, (int)con, res); break;
1929         case lir_sub:  __ sub  (lreg, (int)con, res); break;
1930         case lir_mul:  __ mulx (lreg, (int)con, res); break;
1931         default: ShouldNotReachHere();
1932       }
1933     }
1934   }
1935 }
1936 
1937 
1938 void LIR_Assembler::fpop() {
1939   // do nothing
1940 }
1941 
1942 
1943 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr thread, LIR_Opr dest, LIR_Op* op) {
1944   switch (code) {
1945     case lir_sin:
1946     case lir_tan:
1947     case lir_cos: {
1948       assert(thread->is_valid(), "preserve the thread object for performance reasons");
1949       assert(dest->as_double_reg() == F0, "the result will be in f0/f1");
1950       break;
1951     }
1952     case lir_sqrt: {
1953       assert(!thread->is_valid(), "there is no need for a thread_reg for dsqrt");
1954       FloatRegister src_reg = value->as_double_reg();
1955       FloatRegister dst_reg = dest->as_double_reg();
1956       __ fsqrt(FloatRegisterImpl::D, src_reg, dst_reg);
1957       break;
1958     }
1959     case lir_abs: {
1960       assert(!thread->is_valid(), "there is no need for a thread_reg for fabs");
1961       FloatRegister src_reg = value->as_double_reg();
1962       FloatRegister dst_reg = dest->as_double_reg();
1963       __ fabs(FloatRegisterImpl::D, src_reg, dst_reg);
1964       break;
1965     }
1966     default: {
1967       ShouldNotReachHere();
1968       break;
1969     }
1970   }
1971 }
1972 
1973 
1974 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dest) {
1975   if (right->is_constant()) {
1976     if (dest->is_single_cpu()) {
1977       int simm13 = right->as_constant_ptr()->as_jint();
1978       switch (code) {
1979         case lir_logic_and:   __ and3 (left->as_register(), simm13, dest->as_register()); break;
1980         case lir_logic_or:    __ or3  (left->as_register(), simm13, dest->as_register()); break;
1981         case lir_logic_xor:   __ xor3 (left->as_register(), simm13, dest->as_register()); break;
1982         default: ShouldNotReachHere();
1983       }
1984     } else {
1985       long c = right->as_constant_ptr()->as_jlong();
1986       assert(c == (int)c && Assembler::is_simm13(c), "out of range");
1987       int simm13 = (int)c;
1988       switch (code) {
1989         case lir_logic_and:
1990 #ifndef _LP64
1991           __ and3 (left->as_register_hi(), 0,      dest->as_register_hi());
1992 #endif
1993           __ and3 (left->as_register_lo(), simm13, dest->as_register_lo());
1994           break;
1995 
1996         case lir_logic_or:
1997 #ifndef _LP64
1998           __ or3 (left->as_register_hi(), 0,      dest->as_register_hi());
1999 #endif
2000           __ or3 (left->as_register_lo(), simm13, dest->as_register_lo());
2001           break;
2002 
2003         case lir_logic_xor:
2004 #ifndef _LP64
2005           __ xor3 (left->as_register_hi(), 0,      dest->as_register_hi());
2006 #endif
2007           __ xor3 (left->as_register_lo(), simm13, dest->as_register_lo());
2008           break;
2009 
2010         default: ShouldNotReachHere();
2011       }
2012     }
2013   } else {
2014     assert(right->is_register(), "right should be in register");
2015 
2016     if (dest->is_single_cpu()) {
2017       switch (code) {
2018         case lir_logic_and:   __ and3 (left->as_register(), right->as_register(), dest->as_register()); break;
2019         case lir_logic_or:    __ or3  (left->as_register(), right->as_register(), dest->as_register()); break;
2020         case lir_logic_xor:   __ xor3 (left->as_register(), right->as_register(), dest->as_register()); break;
2021         default: ShouldNotReachHere();
2022       }
2023     } else {
2024 #ifdef _LP64
2025       Register l = (left->is_single_cpu() && left->is_oop_register()) ? left->as_register() :
2026                                                                         left->as_register_lo();
2027       Register r = (right->is_single_cpu() && right->is_oop_register()) ? right->as_register() :
2028                                                                           right->as_register_lo();
2029 
2030       switch (code) {
2031         case lir_logic_and: __ and3 (l, r, dest->as_register_lo()); break;
2032         case lir_logic_or:  __ or3  (l, r, dest->as_register_lo()); break;
2033         case lir_logic_xor: __ xor3 (l, r, dest->as_register_lo()); break;
2034         default: ShouldNotReachHere();
2035       }
2036 #else
2037       switch (code) {
2038         case lir_logic_and:
2039           __ and3 (left->as_register_hi(), right->as_register_hi(), dest->as_register_hi());
2040           __ and3 (left->as_register_lo(), right->as_register_lo(), dest->as_register_lo());
2041           break;
2042 
2043         case lir_logic_or:
2044           __ or3 (left->as_register_hi(), right->as_register_hi(), dest->as_register_hi());
2045           __ or3 (left->as_register_lo(), right->as_register_lo(), dest->as_register_lo());
2046           break;
2047 
2048         case lir_logic_xor:
2049           __ xor3 (left->as_register_hi(), right->as_register_hi(), dest->as_register_hi());
2050           __ xor3 (left->as_register_lo(), right->as_register_lo(), dest->as_register_lo());
2051           break;
2052 
2053         default: ShouldNotReachHere();
2054       }
2055 #endif
2056     }
2057   }
2058 }
2059 
2060 
2061 int LIR_Assembler::shift_amount(BasicType t) {
2062   int elem_size = type2aelembytes(t);
2063   switch (elem_size) {
2064     case 1 : return 0;
2065     case 2 : return 1;
2066     case 4 : return 2;
2067     case 8 : return 3;
2068   }
2069   ShouldNotReachHere();
2070   return -1;
2071 }
2072 
2073 
2074 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) {
2075   assert(exceptionOop->as_register() == Oexception, "should match");
2076   assert(exceptionPC->as_register() == Oissuing_pc, "should match");
2077 
2078   info->add_register_oop(exceptionOop);
2079 
2080   // reuse the debug info from the safepoint poll for the throw op itself
2081   address pc_for_athrow  = __ pc();
2082   int pc_for_athrow_offset = __ offset();
2083   RelocationHolder rspec = internal_word_Relocation::spec(pc_for_athrow);
2084   __ set(pc_for_athrow, Oissuing_pc, rspec);
2085   add_call_info(pc_for_athrow_offset, info); // for exception handler
2086 
2087   __ call(Runtime1::entry_for(Runtime1::handle_exception_id), relocInfo::runtime_call_type);
2088   __ delayed()->nop();
2089 }
2090 
2091 
2092 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) {
2093   assert(exceptionOop->as_register() == Oexception, "should match");
2094 
2095   __ br(Assembler::always, false, Assembler::pt, _unwind_handler_entry);
2096   __ delayed()->nop();
2097 }
2098 
2099 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2100   Register src = op->src()->as_register();
2101   Register dst = op->dst()->as_register();
2102   Register src_pos = op->src_pos()->as_register();
2103   Register dst_pos = op->dst_pos()->as_register();
2104   Register length  = op->length()->as_register();
2105   Register tmp = op->tmp()->as_register();
2106   Register tmp2 = O7;
2107 
2108   int flags = op->flags();
2109   ciArrayKlass* default_type = op->expected_type();
2110   BasicType basic_type = default_type != NULL ? default_type->element_type()->basic_type() : T_ILLEGAL;
2111   if (basic_type == T_ARRAY) basic_type = T_OBJECT;
2112 
2113 #ifdef _LP64
2114   // higher 32bits must be null
2115   __ sra(dst_pos, 0, dst_pos);
2116   __ sra(src_pos, 0, src_pos);
2117   __ sra(length, 0, length);
2118 #endif
2119 
2120   // set up the arraycopy stub information
2121   ArrayCopyStub* stub = op->stub();
2122 
2123   // always do stub if no type information is available.  it's ok if
2124   // the known type isn't loaded since the code sanity checks
2125   // in debug mode and the type isn't required when we know the exact type
2126   // also check that the type is an array type.
2127   if (op->expected_type() == NULL) {
2128     __ mov(src,     O0);
2129     __ mov(src_pos, O1);
2130     __ mov(dst,     O2);
2131     __ mov(dst_pos, O3);
2132     __ mov(length,  O4);
2133     address copyfunc_addr = StubRoutines::generic_arraycopy();
2134 
2135     if (copyfunc_addr == NULL) { // Use C version if stub was not generated
2136       __ call_VM_leaf(tmp, CAST_FROM_FN_PTR(address, Runtime1::arraycopy));
2137     } else {
2138 #ifndef PRODUCT
2139       if (PrintC1Statistics) {
2140         address counter = (address)&Runtime1::_generic_arraycopystub_cnt;
2141         __ inc_counter(counter, G1, G3);
2142       }
2143 #endif
2144       __ call_VM_leaf(tmp, copyfunc_addr);
2145     }
2146 
2147     if (copyfunc_addr != NULL) {
2148       __ xor3(O0, -1, tmp);
2149       __ sub(length, tmp, length);
2150       __ add(src_pos, tmp, src_pos);
2151       __ cmp_zero_and_br(Assembler::less, O0, *stub->entry());
2152       __ delayed()->add(dst_pos, tmp, dst_pos);
2153     } else {
2154       __ cmp_zero_and_br(Assembler::less, O0, *stub->entry());
2155       __ delayed()->nop();
2156     }
2157     __ bind(*stub->continuation());
2158     return;
2159   }
2160 
2161   assert(default_type != NULL && default_type->is_array_klass(), "must be true at this point");
2162 
2163   // make sure src and dst are non-null and load array length
2164   if (flags & LIR_OpArrayCopy::src_null_check) {
2165     __ tst(src);
2166     __ brx(Assembler::equal, false, Assembler::pn, *stub->entry());
2167     __ delayed()->nop();
2168   }
2169 
2170   if (flags & LIR_OpArrayCopy::dst_null_check) {
2171     __ tst(dst);
2172     __ brx(Assembler::equal, false, Assembler::pn, *stub->entry());
2173     __ delayed()->nop();
2174   }
2175 
2176   // If the compiler was not able to prove that exact type of the source or the destination
2177   // of the arraycopy is an array type, check at runtime if the source or the destination is
2178   // an instance type.
2179   if (flags & LIR_OpArrayCopy::type_check) {
2180     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::dst_objarray)) {
2181       __ load_klass(dst, tmp);
2182       __ lduw(tmp, in_bytes(Klass::layout_helper_offset()), tmp2);
2183       __ cmp(tmp2, Klass::_lh_neutral_value);
2184       __ br(Assembler::greaterEqual, false, Assembler::pn, *stub->entry());
2185       __ delayed()->nop();
2186     }
2187 
2188     if (!(flags & LIR_OpArrayCopy::LIR_OpArrayCopy::src_objarray)) {
2189       __ load_klass(src, tmp);
2190       __ lduw(tmp, in_bytes(Klass::layout_helper_offset()), tmp2);
2191       __ cmp(tmp2, Klass::_lh_neutral_value);
2192       __ br(Assembler::greaterEqual, false, Assembler::pn, *stub->entry());
2193       __ delayed()->nop();
2194     }
2195   }
2196 
2197   if (flags & LIR_OpArrayCopy::src_pos_positive_check) {
2198     // test src_pos register
2199     __ cmp_zero_and_br(Assembler::less, src_pos, *stub->entry());
2200     __ delayed()->nop();
2201   }
2202 
2203   if (flags & LIR_OpArrayCopy::dst_pos_positive_check) {
2204     // test dst_pos register
2205     __ cmp_zero_and_br(Assembler::less, dst_pos, *stub->entry());
2206     __ delayed()->nop();
2207   }
2208 
2209   if (flags & LIR_OpArrayCopy::length_positive_check) {
2210     // make sure length isn't negative
2211     __ cmp_zero_and_br(Assembler::less, length, *stub->entry());
2212     __ delayed()->nop();
2213   }
2214 
2215   if (flags & LIR_OpArrayCopy::src_range_check) {
2216     __ ld(src, arrayOopDesc::length_offset_in_bytes(), tmp2);
2217     __ add(length, src_pos, tmp);
2218     __ cmp(tmp2, tmp);
2219     __ br(Assembler::carrySet, false, Assembler::pn, *stub->entry());
2220     __ delayed()->nop();
2221   }
2222 
2223   if (flags & LIR_OpArrayCopy::dst_range_check) {
2224     __ ld(dst, arrayOopDesc::length_offset_in_bytes(), tmp2);
2225     __ add(length, dst_pos, tmp);
2226     __ cmp(tmp2, tmp);
2227     __ br(Assembler::carrySet, false, Assembler::pn, *stub->entry());
2228     __ delayed()->nop();
2229   }
2230 
2231   int shift = shift_amount(basic_type);
2232 
2233   if (flags & LIR_OpArrayCopy::type_check) {
2234     // We don't know the array types are compatible
2235     if (basic_type != T_OBJECT) {
2236       // Simple test for basic type arrays
2237       if (UseCompressedClassPointers) {
2238         // We don't need decode because we just need to compare
2239         __ lduw(src, oopDesc::klass_offset_in_bytes(), tmp);
2240         __ lduw(dst, oopDesc::klass_offset_in_bytes(), tmp2);
2241         __ cmp(tmp, tmp2);
2242         __ br(Assembler::notEqual, false, Assembler::pt, *stub->entry());
2243       } else {
2244         __ ld_ptr(src, oopDesc::klass_offset_in_bytes(), tmp);
2245         __ ld_ptr(dst, oopDesc::klass_offset_in_bytes(), tmp2);
2246         __ cmp(tmp, tmp2);
2247         __ brx(Assembler::notEqual, false, Assembler::pt, *stub->entry());
2248       }
2249       __ delayed()->nop();
2250     } else {
2251       // For object arrays, if src is a sub class of dst then we can
2252       // safely do the copy.
2253       address copyfunc_addr = StubRoutines::checkcast_arraycopy();
2254 
2255       Label cont, slow;
2256       assert_different_registers(tmp, tmp2, G3, G1);
2257 
2258       __ load_klass(src, G3);
2259       __ load_klass(dst, G1);
2260 
2261       __ check_klass_subtype_fast_path(G3, G1, tmp, tmp2, &cont, copyfunc_addr == NULL ? stub->entry() : &slow, NULL);
2262 
2263       __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
2264       __ delayed()->nop();
2265 
2266       __ cmp(G3, 0);
2267       if (copyfunc_addr != NULL) { // use stub if available
2268         // src is not a sub class of dst so we have to do a
2269         // per-element check.
2270         __ br(Assembler::notEqual, false, Assembler::pt, cont);
2271         __ delayed()->nop();
2272 
2273         __ bind(slow);
2274 
2275         int mask = LIR_OpArrayCopy::src_objarray|LIR_OpArrayCopy::dst_objarray;
2276         if ((flags & mask) != mask) {
2277           // Check that at least both of them object arrays.
2278           assert(flags & mask, "one of the two should be known to be an object array");
2279 
2280           if (!(flags & LIR_OpArrayCopy::src_objarray)) {
2281             __ load_klass(src, tmp);
2282           } else if (!(flags & LIR_OpArrayCopy::dst_objarray)) {
2283             __ load_klass(dst, tmp);
2284           }
2285           int lh_offset = in_bytes(Klass::layout_helper_offset());
2286 
2287           __ lduw(tmp, lh_offset, tmp2);
2288 
2289           jint objArray_lh = Klass::array_layout_helper(T_OBJECT);
2290           __ set(objArray_lh, tmp);
2291           __ cmp(tmp, tmp2);
2292           __ br(Assembler::notEqual, false, Assembler::pt,  *stub->entry());
2293           __ delayed()->nop();
2294         }
2295 
2296         Register src_ptr = O0;
2297         Register dst_ptr = O1;
2298         Register len     = O2;
2299         Register chk_off = O3;
2300         Register super_k = O4;
2301 
2302         __ add(src, arrayOopDesc::base_offset_in_bytes(basic_type), src_ptr);
2303         if (shift == 0) {
2304           __ add(src_ptr, src_pos, src_ptr);
2305         } else {
2306           __ sll(src_pos, shift, tmp);
2307           __ add(src_ptr, tmp, src_ptr);
2308         }
2309 
2310         __ add(dst, arrayOopDesc::base_offset_in_bytes(basic_type), dst_ptr);
2311         if (shift == 0) {
2312           __ add(dst_ptr, dst_pos, dst_ptr);
2313         } else {
2314           __ sll(dst_pos, shift, tmp);
2315           __ add(dst_ptr, tmp, dst_ptr);
2316         }
2317         __ mov(length, len);
2318         __ load_klass(dst, tmp);
2319 
2320         int ek_offset = in_bytes(ObjArrayKlass::element_klass_offset());
2321         __ ld_ptr(tmp, ek_offset, super_k);
2322 
2323         int sco_offset = in_bytes(Klass::super_check_offset_offset());
2324         __ lduw(super_k, sco_offset, chk_off);
2325 
2326         __ call_VM_leaf(tmp, copyfunc_addr);
2327 
2328 #ifndef PRODUCT
2329         if (PrintC1Statistics) {
2330           Label failed;
2331           __ br_notnull_short(O0, Assembler::pn, failed);
2332           __ inc_counter((address)&Runtime1::_arraycopy_checkcast_cnt, G1, G3);
2333           __ bind(failed);
2334         }
2335 #endif
2336 
2337         __ br_null(O0, false, Assembler::pt,  *stub->continuation());
2338         __ delayed()->xor3(O0, -1, tmp);
2339 
2340 #ifndef PRODUCT
2341         if (PrintC1Statistics) {
2342           __ inc_counter((address)&Runtime1::_arraycopy_checkcast_attempt_cnt, G1, G3);
2343         }
2344 #endif
2345 
2346         __ sub(length, tmp, length);
2347         __ add(src_pos, tmp, src_pos);
2348         __ br(Assembler::always, false, Assembler::pt, *stub->entry());
2349         __ delayed()->add(dst_pos, tmp, dst_pos);
2350 
2351         __ bind(cont);
2352       } else {
2353         __ br(Assembler::equal, false, Assembler::pn, *stub->entry());
2354         __ delayed()->nop();
2355         __ bind(cont);
2356       }
2357     }
2358   }
2359 
2360 #ifdef ASSERT
2361   if (basic_type != T_OBJECT || !(flags & LIR_OpArrayCopy::type_check)) {
2362     // Sanity check the known type with the incoming class.  For the
2363     // primitive case the types must match exactly with src.klass and
2364     // dst.klass each exactly matching the default type.  For the
2365     // object array case, if no type check is needed then either the
2366     // dst type is exactly the expected type and the src type is a
2367     // subtype which we can't check or src is the same array as dst
2368     // but not necessarily exactly of type default_type.
2369     Label known_ok, halt;
2370     metadata2reg(op->expected_type()->constant_encoding(), tmp);
2371     if (UseCompressedClassPointers) {
2372       // tmp holds the default type. It currently comes uncompressed after the
2373       // load of a constant, so encode it.
2374       __ encode_klass_not_null(tmp);
2375       // load the raw value of the dst klass, since we will be comparing
2376       // uncompressed values directly.
2377       __ lduw(dst, oopDesc::klass_offset_in_bytes(), tmp2);
2378       if (basic_type != T_OBJECT) {
2379         __ cmp(tmp, tmp2);
2380         __ br(Assembler::notEqual, false, Assembler::pn, halt);
2381         // load the raw value of the src klass.
2382         __ delayed()->lduw(src, oopDesc::klass_offset_in_bytes(), tmp2);
2383         __ cmp_and_br_short(tmp, tmp2, Assembler::equal, Assembler::pn, known_ok);
2384       } else {
2385         __ cmp(tmp, tmp2);
2386         __ br(Assembler::equal, false, Assembler::pn, known_ok);
2387         __ delayed()->cmp(src, dst);
2388         __ brx(Assembler::equal, false, Assembler::pn, known_ok);
2389         __ delayed()->nop();
2390       }
2391     } else {
2392       __ ld_ptr(dst, oopDesc::klass_offset_in_bytes(), tmp2);
2393       if (basic_type != T_OBJECT) {
2394         __ cmp(tmp, tmp2);
2395         __ brx(Assembler::notEqual, false, Assembler::pn, halt);
2396         __ delayed()->ld_ptr(src, oopDesc::klass_offset_in_bytes(), tmp2);
2397         __ cmp_and_brx_short(tmp, tmp2, Assembler::equal, Assembler::pn, known_ok);
2398       } else {
2399         __ cmp(tmp, tmp2);
2400         __ brx(Assembler::equal, false, Assembler::pn, known_ok);
2401         __ delayed()->cmp(src, dst);
2402         __ brx(Assembler::equal, false, Assembler::pn, known_ok);
2403         __ delayed()->nop();
2404       }
2405     }
2406     __ bind(halt);
2407     __ stop("incorrect type information in arraycopy");
2408     __ bind(known_ok);
2409   }
2410 #endif
2411 
2412 #ifndef PRODUCT
2413   if (PrintC1Statistics) {
2414     address counter = Runtime1::arraycopy_count_address(basic_type);
2415     __ inc_counter(counter, G1, G3);
2416   }
2417 #endif
2418 
2419   Register src_ptr = O0;
2420   Register dst_ptr = O1;
2421   Register len     = O2;
2422 
2423   __ add(src, arrayOopDesc::base_offset_in_bytes(basic_type), src_ptr);
2424   if (shift == 0) {
2425     __ add(src_ptr, src_pos, src_ptr);
2426   } else {
2427     __ sll(src_pos, shift, tmp);
2428     __ add(src_ptr, tmp, src_ptr);
2429   }
2430 
2431   __ add(dst, arrayOopDesc::base_offset_in_bytes(basic_type), dst_ptr);
2432   if (shift == 0) {
2433     __ add(dst_ptr, dst_pos, dst_ptr);
2434   } else {
2435     __ sll(dst_pos, shift, tmp);
2436     __ add(dst_ptr, tmp, dst_ptr);
2437   }
2438 
2439   bool disjoint = (flags & LIR_OpArrayCopy::overlapping) == 0;
2440   bool aligned = (flags & LIR_OpArrayCopy::unaligned) == 0;
2441   const char *name;
2442   address entry = StubRoutines::select_arraycopy_function(basic_type, aligned, disjoint, name, false);
2443 
2444   // arraycopy stubs takes a length in number of elements, so don't scale it.
2445   __ mov(length, len);
2446   __ call_VM_leaf(tmp, entry);
2447 
2448   __ bind(*stub->continuation());
2449 }
2450 
2451 
2452 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) {
2453   if (dest->is_single_cpu()) {
2454 #ifdef _LP64
2455     if (left->type() == T_OBJECT) {
2456       switch (code) {
2457         case lir_shl:  __ sllx  (left->as_register(), count->as_register(), dest->as_register()); break;
2458         case lir_shr:  __ srax  (left->as_register(), count->as_register(), dest->as_register()); break;
2459         case lir_ushr: __ srl   (left->as_register(), count->as_register(), dest->as_register()); break;
2460         default: ShouldNotReachHere();
2461       }
2462     } else
2463 #endif
2464       switch (code) {
2465         case lir_shl:  __ sll   (left->as_register(), count->as_register(), dest->as_register()); break;
2466         case lir_shr:  __ sra   (left->as_register(), count->as_register(), dest->as_register()); break;
2467         case lir_ushr: __ srl   (left->as_register(), count->as_register(), dest->as_register()); break;
2468         default: ShouldNotReachHere();
2469       }
2470   } else {
2471 #ifdef _LP64
2472     switch (code) {
2473       case lir_shl:  __ sllx  (left->as_register_lo(), count->as_register(), dest->as_register_lo()); break;
2474       case lir_shr:  __ srax  (left->as_register_lo(), count->as_register(), dest->as_register_lo()); break;
2475       case lir_ushr: __ srlx  (left->as_register_lo(), count->as_register(), dest->as_register_lo()); break;
2476       default: ShouldNotReachHere();
2477     }
2478 #else
2479     switch (code) {
2480       case lir_shl:  __ lshl  (left->as_register_hi(), left->as_register_lo(), count->as_register(), dest->as_register_hi(), dest->as_register_lo(), G3_scratch); break;
2481       case lir_shr:  __ lshr  (left->as_register_hi(), left->as_register_lo(), count->as_register(), dest->as_register_hi(), dest->as_register_lo(), G3_scratch); break;
2482       case lir_ushr: __ lushr (left->as_register_hi(), left->as_register_lo(), count->as_register(), dest->as_register_hi(), dest->as_register_lo(), G3_scratch); break;
2483       default: ShouldNotReachHere();
2484     }
2485 #endif
2486   }
2487 }
2488 
2489 
2490 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) {
2491 #ifdef _LP64
2492   if (left->type() == T_OBJECT) {
2493     count = count & 63;  // shouldn't shift by more than sizeof(intptr_t)
2494     Register l = left->as_register();
2495     Register d = dest->as_register_lo();
2496     switch (code) {
2497       case lir_shl:  __ sllx  (l, count, d); break;
2498       case lir_shr:  __ srax  (l, count, d); break;
2499       case lir_ushr: __ srlx  (l, count, d); break;
2500       default: ShouldNotReachHere();
2501     }
2502     return;
2503   }
2504 #endif
2505 
2506   if (dest->is_single_cpu()) {
2507     count = count & 0x1F; // Java spec
2508     switch (code) {
2509       case lir_shl:  __ sll   (left->as_register(), count, dest->as_register()); break;
2510       case lir_shr:  __ sra   (left->as_register(), count, dest->as_register()); break;
2511       case lir_ushr: __ srl   (left->as_register(), count, dest->as_register()); break;
2512       default: ShouldNotReachHere();
2513     }
2514   } else if (dest->is_double_cpu()) {
2515     count = count & 63; // Java spec
2516     switch (code) {
2517       case lir_shl:  __ sllx  (left->as_pointer_register(), count, dest->as_pointer_register()); break;
2518       case lir_shr:  __ srax  (left->as_pointer_register(), count, dest->as_pointer_register()); break;
2519       case lir_ushr: __ srlx  (left->as_pointer_register(), count, dest->as_pointer_register()); break;
2520       default: ShouldNotReachHere();
2521     }
2522   } else {
2523     ShouldNotReachHere();
2524   }
2525 }
2526 
2527 
2528 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) {
2529   assert(op->tmp1()->as_register()  == G1 &&
2530          op->tmp2()->as_register()  == G3 &&
2531          op->tmp3()->as_register()  == G4 &&
2532          op->obj()->as_register()   == O0 &&
2533          op->klass()->as_register() == G5, "must be");
2534   if (op->init_check()) {
2535     __ ldub(op->klass()->as_register(),
2536           in_bytes(InstanceKlass::init_state_offset()),
2537           op->tmp1()->as_register());
2538     add_debug_info_for_null_check_here(op->stub()->info());
2539     __ cmp(op->tmp1()->as_register(), InstanceKlass::fully_initialized);
2540     __ br(Assembler::notEqual, false, Assembler::pn, *op->stub()->entry());
2541     __ delayed()->nop();
2542   }
2543   __ allocate_object(op->obj()->as_register(),
2544                      op->tmp1()->as_register(),
2545                      op->tmp2()->as_register(),
2546                      op->tmp3()->as_register(),
2547                      op->header_size(),
2548                      op->object_size(),
2549                      op->klass()->as_register(),
2550                      *op->stub()->entry());
2551   __ bind(*op->stub()->continuation());
2552   __ verify_oop(op->obj()->as_register());
2553 }
2554 
2555 
2556 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
2557   assert(op->tmp1()->as_register()  == G1 &&
2558          op->tmp2()->as_register()  == G3 &&
2559          op->tmp3()->as_register()  == G4 &&
2560          op->tmp4()->as_register()  == O1 &&
2561          op->klass()->as_register() == G5, "must be");
2562 
2563   LP64_ONLY( __ signx(op->len()->as_register()); )
2564   if (UseSlowPath ||
2565       (!UseFastNewObjectArray && (op->type() == T_OBJECT || op->type() == T_ARRAY)) ||
2566       (!UseFastNewTypeArray   && (op->type() != T_OBJECT && op->type() != T_ARRAY))) {
2567     __ br(Assembler::always, false, Assembler::pt, *op->stub()->entry());
2568     __ delayed()->nop();
2569   } else {
2570     __ allocate_array(op->obj()->as_register(),
2571                       op->len()->as_register(),
2572                       op->tmp1()->as_register(),
2573                       op->tmp2()->as_register(),
2574                       op->tmp3()->as_register(),
2575                       arrayOopDesc::header_size(op->type()),
2576                       type2aelembytes(op->type()),
2577                       op->klass()->as_register(),
2578                       *op->stub()->entry());
2579   }
2580   __ bind(*op->stub()->continuation());
2581 }
2582 
2583 
2584 void LIR_Assembler::type_profile_helper(Register mdo, int mdo_offset_bias,
2585                                         ciMethodData *md, ciProfileData *data,
2586                                         Register recv, Register tmp1, Label* update_done) {
2587   uint i;
2588   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2589     Label next_test;
2590     // See if the receiver is receiver[n].
2591     Address receiver_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
2592                           mdo_offset_bias);
2593     __ ld_ptr(receiver_addr, tmp1);
2594     __ verify_klass_ptr(tmp1);
2595     __ cmp_and_brx_short(recv, tmp1, Assembler::notEqual, Assembler::pt, next_test);
2596     Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
2597                       mdo_offset_bias);
2598     __ ld_ptr(data_addr, tmp1);
2599     __ add(tmp1, DataLayout::counter_increment, tmp1);
2600     __ st_ptr(tmp1, data_addr);
2601     __ ba(*update_done);
2602     __ delayed()->nop();
2603     __ bind(next_test);
2604   }
2605 
2606   // Didn't find receiver; find next empty slot and fill it in
2607   for (i = 0; i < VirtualCallData::row_limit(); i++) {
2608     Label next_test;
2609     Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)) -
2610                       mdo_offset_bias);
2611     __ ld_ptr(recv_addr, tmp1);
2612     __ br_notnull_short(tmp1, Assembler::pt, next_test);
2613     __ st_ptr(recv, recv_addr);
2614     __ set(DataLayout::counter_increment, tmp1);
2615     __ st_ptr(tmp1, mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)) -
2616               mdo_offset_bias);
2617     __ ba(*update_done);
2618     __ delayed()->nop();
2619     __ bind(next_test);
2620   }
2621 }
2622 
2623 
2624 void LIR_Assembler::setup_md_access(ciMethod* method, int bci,
2625                                     ciMethodData*& md, ciProfileData*& data, int& mdo_offset_bias) {
2626   md = method->method_data_or_null();
2627   assert(md != NULL, "Sanity");
2628   data = md->bci_to_data(bci);
2629   assert(data != NULL,       "need data for checkcast");
2630   assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
2631   if (!Assembler::is_simm13(md->byte_offset_of_slot(data, DataLayout::header_offset()) + data->size_in_bytes())) {
2632     // The offset is large so bias the mdo by the base of the slot so
2633     // that the ld can use simm13s to reference the slots of the data
2634     mdo_offset_bias = md->byte_offset_of_slot(data, DataLayout::header_offset());
2635   }
2636 }
2637 
2638 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) {
2639   // we always need a stub for the failure case.
2640   CodeStub* stub = op->stub();
2641   Register obj = op->object()->as_register();
2642   Register k_RInfo = op->tmp1()->as_register();
2643   Register klass_RInfo = op->tmp2()->as_register();
2644   Register dst = op->result_opr()->as_register();
2645   Register Rtmp1 = op->tmp3()->as_register();
2646   ciKlass* k = op->klass();
2647 
2648 
2649   if (obj == k_RInfo) {
2650     k_RInfo = klass_RInfo;
2651     klass_RInfo = obj;
2652   }
2653 
2654   ciMethodData* md;
2655   ciProfileData* data;
2656   int mdo_offset_bias = 0;
2657   if (op->should_profile()) {
2658     ciMethod* method = op->profiled_method();
2659     assert(method != NULL, "Should have method");
2660     setup_md_access(method, op->profiled_bci(), md, data, mdo_offset_bias);
2661 
2662     Label not_null;
2663     __ br_notnull_short(obj, Assembler::pn, not_null);
2664     Register mdo      = k_RInfo;
2665     Register data_val = Rtmp1;
2666     metadata2reg(md->constant_encoding(), mdo);
2667     if (mdo_offset_bias > 0) {
2668       __ set(mdo_offset_bias, data_val);
2669       __ add(mdo, data_val, mdo);
2670     }
2671     Address flags_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias);
2672     __ ldub(flags_addr, data_val);
2673     __ or3(data_val, BitData::null_seen_byte_constant(), data_val);
2674     __ stb(data_val, flags_addr);
2675     __ ba(*obj_is_null);
2676     __ delayed()->nop();
2677     __ bind(not_null);
2678   } else {
2679     __ br_null(obj, false, Assembler::pn, *obj_is_null);
2680     __ delayed()->nop();
2681   }
2682 
2683   Label profile_cast_failure, profile_cast_success;
2684   Label *failure_target = op->should_profile() ? &profile_cast_failure : failure;
2685   Label *success_target = op->should_profile() ? &profile_cast_success : success;
2686 
2687   // patching may screw with our temporaries on sparc,
2688   // so let's do it before loading the class
2689   if (k->is_loaded()) {
2690     metadata2reg(k->constant_encoding(), k_RInfo);
2691   } else {
2692     klass2reg_with_patching(k_RInfo, op->info_for_patch());
2693   }
2694   assert(obj != k_RInfo, "must be different");
2695 
2696   // get object class
2697   // not a safepoint as obj null check happens earlier
2698   __ load_klass(obj, klass_RInfo);
2699   if (op->fast_check()) {
2700     assert_different_registers(klass_RInfo, k_RInfo);
2701     __ cmp(k_RInfo, klass_RInfo);
2702     __ brx(Assembler::notEqual, false, Assembler::pt, *failure_target);
2703     __ delayed()->nop();
2704   } else {
2705     bool need_slow_path = true;
2706     if (k->is_loaded()) {
2707       if ((int) k->super_check_offset() != in_bytes(Klass::secondary_super_cache_offset()))
2708         need_slow_path = false;
2709       // perform the fast part of the checking logic
2710       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, noreg,
2711                                        (need_slow_path ? success_target : NULL),
2712                                        failure_target, NULL,
2713                                        RegisterOrConstant(k->super_check_offset()));
2714     } else {
2715       // perform the fast part of the checking logic
2716       __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, O7, success_target,
2717                                        failure_target, NULL);
2718     }
2719     if (need_slow_path) {
2720       // call out-of-line instance of __ check_klass_subtype_slow_path(...):
2721       assert(klass_RInfo == G3 && k_RInfo == G1, "incorrect call setup");
2722       __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
2723       __ delayed()->nop();
2724       __ cmp(G3, 0);
2725       __ br(Assembler::equal, false, Assembler::pn, *failure_target);
2726       __ delayed()->nop();
2727       // Fall through to success case
2728     }
2729   }
2730 
2731   if (op->should_profile()) {
2732     Register mdo  = klass_RInfo, recv = k_RInfo, tmp1 = Rtmp1;
2733     assert_different_registers(obj, mdo, recv, tmp1);
2734     __ bind(profile_cast_success);
2735     metadata2reg(md->constant_encoding(), mdo);
2736     if (mdo_offset_bias > 0) {
2737       __ set(mdo_offset_bias, tmp1);
2738       __ add(mdo, tmp1, mdo);
2739     }
2740     __ load_klass(obj, recv);
2741     type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, success);
2742     // Jump over the failure case
2743     __ ba(*success);
2744     __ delayed()->nop();
2745     // Cast failure case
2746     __ bind(profile_cast_failure);
2747     metadata2reg(md->constant_encoding(), mdo);
2748     if (mdo_offset_bias > 0) {
2749       __ set(mdo_offset_bias, tmp1);
2750       __ add(mdo, tmp1, mdo);
2751     }
2752     Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
2753     __ ld_ptr(data_addr, tmp1);
2754     __ sub(tmp1, DataLayout::counter_increment, tmp1);
2755     __ st_ptr(tmp1, data_addr);
2756     __ ba(*failure);
2757     __ delayed()->nop();
2758   }
2759   __ ba(*success);
2760   __ delayed()->nop();
2761 }
2762 
2763 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) {
2764   LIR_Code code = op->code();
2765   if (code == lir_store_check) {
2766     Register value = op->object()->as_register();
2767     Register array = op->array()->as_register();
2768     Register k_RInfo = op->tmp1()->as_register();
2769     Register klass_RInfo = op->tmp2()->as_register();
2770     Register Rtmp1 = op->tmp3()->as_register();
2771 
2772     __ verify_oop(value);
2773     CodeStub* stub = op->stub();
2774     // check if it needs to be profiled
2775     ciMethodData* md;
2776     ciProfileData* data;
2777     int mdo_offset_bias = 0;
2778     if (op->should_profile()) {
2779       ciMethod* method = op->profiled_method();
2780       assert(method != NULL, "Should have method");
2781       setup_md_access(method, op->profiled_bci(), md, data, mdo_offset_bias);
2782     }
2783     Label profile_cast_success, profile_cast_failure, done;
2784     Label *success_target = op->should_profile() ? &profile_cast_success : &done;
2785     Label *failure_target = op->should_profile() ? &profile_cast_failure : stub->entry();
2786 
2787     if (op->should_profile()) {
2788       Label not_null;
2789       __ br_notnull_short(value, Assembler::pn, not_null);
2790       Register mdo      = k_RInfo;
2791       Register data_val = Rtmp1;
2792       metadata2reg(md->constant_encoding(), mdo);
2793       if (mdo_offset_bias > 0) {
2794         __ set(mdo_offset_bias, data_val);
2795         __ add(mdo, data_val, mdo);
2796       }
2797       Address flags_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()) - mdo_offset_bias);
2798       __ ldub(flags_addr, data_val);
2799       __ or3(data_val, BitData::null_seen_byte_constant(), data_val);
2800       __ stb(data_val, flags_addr);
2801       __ ba_short(done);
2802       __ bind(not_null);
2803     } else {
2804       __ br_null_short(value, Assembler::pn, done);
2805     }
2806     add_debug_info_for_null_check_here(op->info_for_exception());
2807     __ load_klass(array, k_RInfo);
2808     __ load_klass(value, klass_RInfo);
2809 
2810     // get instance klass
2811     __ ld_ptr(Address(k_RInfo, ObjArrayKlass::element_klass_offset()), k_RInfo);
2812     // perform the fast part of the checking logic
2813     __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, O7, success_target, failure_target, NULL);
2814 
2815     // call out-of-line instance of __ check_klass_subtype_slow_path(...):
2816     assert(klass_RInfo == G3 && k_RInfo == G1, "incorrect call setup");
2817     __ call(Runtime1::entry_for(Runtime1::slow_subtype_check_id), relocInfo::runtime_call_type);
2818     __ delayed()->nop();
2819     __ cmp(G3, 0);
2820     __ br(Assembler::equal, false, Assembler::pn, *failure_target);
2821     __ delayed()->nop();
2822     // fall through to the success case
2823 
2824     if (op->should_profile()) {
2825       Register mdo  = klass_RInfo, recv = k_RInfo, tmp1 = Rtmp1;
2826       assert_different_registers(value, mdo, recv, tmp1);
2827       __ bind(profile_cast_success);
2828       metadata2reg(md->constant_encoding(), mdo);
2829       if (mdo_offset_bias > 0) {
2830         __ set(mdo_offset_bias, tmp1);
2831         __ add(mdo, tmp1, mdo);
2832       }
2833       __ load_klass(value, recv);
2834       type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &done);
2835       __ ba_short(done);
2836       // Cast failure case
2837       __ bind(profile_cast_failure);
2838       metadata2reg(md->constant_encoding(), mdo);
2839       if (mdo_offset_bias > 0) {
2840         __ set(mdo_offset_bias, tmp1);
2841         __ add(mdo, tmp1, mdo);
2842       }
2843       Address data_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
2844       __ ld_ptr(data_addr, tmp1);
2845       __ sub(tmp1, DataLayout::counter_increment, tmp1);
2846       __ st_ptr(tmp1, data_addr);
2847       __ ba(*stub->entry());
2848       __ delayed()->nop();
2849     }
2850     __ bind(done);
2851   } else if (code == lir_checkcast) {
2852     Register obj = op->object()->as_register();
2853     Register dst = op->result_opr()->as_register();
2854     Label success;
2855     emit_typecheck_helper(op, &success, op->stub()->entry(), &success);
2856     __ bind(success);
2857     __ mov(obj, dst);
2858   } else if (code == lir_instanceof) {
2859     Register obj = op->object()->as_register();
2860     Register dst = op->result_opr()->as_register();
2861     Label success, failure, done;
2862     emit_typecheck_helper(op, &success, &failure, &failure);
2863     __ bind(failure);
2864     __ set(0, dst);
2865     __ ba_short(done);
2866     __ bind(success);
2867     __ set(1, dst);
2868     __ bind(done);
2869   } else {
2870     ShouldNotReachHere();
2871   }
2872 
2873 }
2874 
2875 
2876 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
2877   if (op->code() == lir_cas_long) {
2878     assert(VM_Version::supports_cx8(), "wrong machine");
2879     Register addr = op->addr()->as_pointer_register();
2880     Register cmp_value_lo = op->cmp_value()->as_register_lo();
2881     Register cmp_value_hi = op->cmp_value()->as_register_hi();
2882     Register new_value_lo = op->new_value()->as_register_lo();
2883     Register new_value_hi = op->new_value()->as_register_hi();
2884     Register t1 = op->tmp1()->as_register();
2885     Register t2 = op->tmp2()->as_register();
2886 #ifdef _LP64
2887     __ mov(cmp_value_lo, t1);
2888     __ mov(new_value_lo, t2);
2889     // perform the compare and swap operation
2890     __ casx(addr, t1, t2);
2891     // generate condition code - if the swap succeeded, t2 ("new value" reg) was
2892     // overwritten with the original value in "addr" and will be equal to t1.
2893     __ cmp(t1, t2);
2894 #else
2895     // move high and low halves of long values into single registers
2896     __ sllx(cmp_value_hi, 32, t1);         // shift high half into temp reg
2897     __ srl(cmp_value_lo, 0, cmp_value_lo); // clear upper 32 bits of low half
2898     __ or3(t1, cmp_value_lo, t1);          // t1 holds 64-bit compare value
2899     __ sllx(new_value_hi, 32, t2);
2900     __ srl(new_value_lo, 0, new_value_lo);
2901     __ or3(t2, new_value_lo, t2);          // t2 holds 64-bit value to swap
2902     // perform the compare and swap operation
2903     __ casx(addr, t1, t2);
2904     // generate condition code - if the swap succeeded, t2 ("new value" reg) was
2905     // overwritten with the original value in "addr" and will be equal to t1.
2906     // Produce icc flag for 32bit.
2907     __ sub(t1, t2, t2);
2908     __ srlx(t2, 32, t1);
2909     __ orcc(t2, t1, G0);
2910 #endif
2911   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj) {
2912     Register addr = op->addr()->as_pointer_register();
2913     Register cmp_value = op->cmp_value()->as_register();
2914     Register new_value = op->new_value()->as_register();
2915     Register t1 = op->tmp1()->as_register();
2916     Register t2 = op->tmp2()->as_register();
2917     __ mov(cmp_value, t1);
2918     __ mov(new_value, t2);
2919     if (op->code() == lir_cas_obj) {
2920       if (UseCompressedOops) {
2921         __ encode_heap_oop(t1);
2922         __ encode_heap_oop(t2);
2923         __ cas(addr, t1, t2);
2924       } else {
2925         __ cas_ptr(addr, t1, t2);
2926       }
2927     } else {
2928       __ cas(addr, t1, t2);
2929     }
2930     __ cmp(t1, t2);
2931   } else {
2932     Unimplemented();
2933   }
2934 }
2935 
2936 void LIR_Assembler::set_24bit_FPU() {
2937   Unimplemented();
2938 }
2939 
2940 
2941 void LIR_Assembler::reset_FPU() {
2942   Unimplemented();
2943 }
2944 
2945 
2946 void LIR_Assembler::breakpoint() {
2947   __ breakpoint_trap();
2948 }
2949 
2950 
2951 void LIR_Assembler::push(LIR_Opr opr) {
2952   Unimplemented();
2953 }
2954 
2955 
2956 void LIR_Assembler::pop(LIR_Opr opr) {
2957   Unimplemented();
2958 }
2959 
2960 
2961 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst_opr) {
2962   Address mon_addr = frame_map()->address_for_monitor_lock(monitor_no);
2963   Register dst = dst_opr->as_register();
2964   Register reg = mon_addr.base();
2965   int offset = mon_addr.disp();
2966   // compute pointer to BasicLock
2967   if (mon_addr.is_simm13()) {
2968     __ add(reg, offset, dst);
2969   } else {
2970     __ set(offset, dst);
2971     __ add(dst, reg, dst);
2972   }
2973 }
2974 
2975 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) {
2976   fatal("CRC32 intrinsic is not implemented on this platform");
2977 }
2978 
2979 void LIR_Assembler::emit_lock(LIR_OpLock* op) {
2980   Register obj = op->obj_opr()->as_register();
2981   Register hdr = op->hdr_opr()->as_register();
2982   Register lock = op->lock_opr()->as_register();
2983 
2984   // obj may not be an oop
2985   if (op->code() == lir_lock) {
2986     MonitorEnterStub* stub = (MonitorEnterStub*)op->stub();
2987     if (UseFastLocking) {
2988       assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
2989       // add debug info for NullPointerException only if one is possible
2990       if (op->info() != NULL) {
2991         add_debug_info_for_null_check_here(op->info());
2992       }
2993       __ lock_object(hdr, obj, lock, op->scratch_opr()->as_register(), *op->stub()->entry());
2994     } else {
2995       // always do slow locking
2996       // note: the slow locking code could be inlined here, however if we use
2997       //       slow locking, speed doesn't matter anyway and this solution is
2998       //       simpler and requires less duplicated code - additionally, the
2999       //       slow locking code is the same in either case which simplifies
3000       //       debugging
3001       __ br(Assembler::always, false, Assembler::pt, *op->stub()->entry());
3002       __ delayed()->nop();
3003     }
3004   } else {
3005     assert (op->code() == lir_unlock, "Invalid code, expected lir_unlock");
3006     if (UseFastLocking) {
3007       assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header");
3008       __ unlock_object(hdr, obj, lock, *op->stub()->entry());
3009     } else {
3010       // always do slow unlocking
3011       // note: the slow unlocking code could be inlined here, however if we use
3012       //       slow unlocking, speed doesn't matter anyway and this solution is
3013       //       simpler and requires less duplicated code - additionally, the
3014       //       slow unlocking code is the same in either case which simplifies
3015       //       debugging
3016       __ br(Assembler::always, false, Assembler::pt, *op->stub()->entry());
3017       __ delayed()->nop();
3018     }
3019   }
3020   __ bind(*op->stub()->continuation());
3021 }
3022 
3023 
3024 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) {
3025   ciMethod* method = op->profiled_method();
3026   int bci          = op->profiled_bci();
3027   ciMethod* callee = op->profiled_callee();
3028 
3029   // Update counter for all call types
3030   ciMethodData* md = method->method_data_or_null();
3031   assert(md != NULL, "Sanity");
3032   ciProfileData* data = md->bci_to_data(bci);
3033   assert(data->is_CounterData(), "need CounterData for calls");
3034   assert(op->mdo()->is_single_cpu(),  "mdo must be allocated");
3035   Register mdo  = op->mdo()->as_register();
3036 #ifdef _LP64
3037   assert(op->tmp1()->is_double_cpu(), "tmp1 must be allocated");
3038   Register tmp1 = op->tmp1()->as_register_lo();
3039 #else
3040   assert(op->tmp1()->is_single_cpu(), "tmp1 must be allocated");
3041   Register tmp1 = op->tmp1()->as_register();
3042 #endif
3043   metadata2reg(md->constant_encoding(), mdo);
3044   int mdo_offset_bias = 0;
3045   if (!Assembler::is_simm13(md->byte_offset_of_slot(data, CounterData::count_offset()) +
3046                             data->size_in_bytes())) {
3047     // The offset is large so bias the mdo by the base of the slot so
3048     // that the ld can use simm13s to reference the slots of the data
3049     mdo_offset_bias = md->byte_offset_of_slot(data, CounterData::count_offset());
3050     __ set(mdo_offset_bias, O7);
3051     __ add(mdo, O7, mdo);
3052   }
3053 
3054   Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()) - mdo_offset_bias);
3055   Bytecodes::Code bc = method->java_code_at_bci(bci);
3056   const bool callee_is_static = callee->is_loaded() && callee->is_static();
3057   // Perform additional virtual call profiling for invokevirtual and
3058   // invokeinterface bytecodes
3059   if ((bc == Bytecodes::_invokevirtual || bc == Bytecodes::_invokeinterface) &&
3060       !callee_is_static &&  // required for optimized MH invokes
3061       C1ProfileVirtualCalls) {
3062     assert(op->recv()->is_single_cpu(), "recv must be allocated");
3063     Register recv = op->recv()->as_register();
3064     assert_different_registers(mdo, tmp1, recv);
3065     assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls");
3066     ciKlass* known_klass = op->known_holder();
3067     if (C1OptimizeVirtualCallProfiling && known_klass != NULL) {
3068       // We know the type that will be seen at this call site; we can
3069       // statically update the MethodData* rather than needing to do
3070       // dynamic tests on the receiver type
3071 
3072       // NOTE: we should probably put a lock around this search to
3073       // avoid collisions by concurrent compilations
3074       ciVirtualCallData* vc_data = (ciVirtualCallData*) data;
3075       uint i;
3076       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3077         ciKlass* receiver = vc_data->receiver(i);
3078         if (known_klass->equals(receiver)) {
3079           Address data_addr(mdo, md->byte_offset_of_slot(data,
3080                                                          VirtualCallData::receiver_count_offset(i)) -
3081                             mdo_offset_bias);
3082           __ ld_ptr(data_addr, tmp1);
3083           __ add(tmp1, DataLayout::counter_increment, tmp1);
3084           __ st_ptr(tmp1, data_addr);
3085           return;
3086         }
3087       }
3088 
3089       // Receiver type not found in profile data; select an empty slot
3090 
3091       // Note that this is less efficient than it should be because it
3092       // always does a write to the receiver part of the
3093       // VirtualCallData rather than just the first time
3094       for (i = 0; i < VirtualCallData::row_limit(); i++) {
3095         ciKlass* receiver = vc_data->receiver(i);
3096         if (receiver == NULL) {
3097           Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i)) -
3098                             mdo_offset_bias);
3099           metadata2reg(known_klass->constant_encoding(), tmp1);
3100           __ st_ptr(tmp1, recv_addr);
3101           Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i)) -
3102                             mdo_offset_bias);
3103           __ ld_ptr(data_addr, tmp1);
3104           __ add(tmp1, DataLayout::counter_increment, tmp1);
3105           __ st_ptr(tmp1, data_addr);
3106           return;
3107         }
3108       }
3109     } else {
3110       __ load_klass(recv, recv);
3111       Label update_done;
3112       type_profile_helper(mdo, mdo_offset_bias, md, data, recv, tmp1, &update_done);
3113       // Receiver did not match any saved receiver and there is no empty row for it.
3114       // Increment total counter to indicate polymorphic case.
3115       __ ld_ptr(counter_addr, tmp1);
3116       __ add(tmp1, DataLayout::counter_increment, tmp1);
3117       __ st_ptr(tmp1, counter_addr);
3118 
3119       __ bind(update_done);
3120     }
3121   } else {
3122     // Static call
3123     __ ld_ptr(counter_addr, tmp1);
3124     __ add(tmp1, DataLayout::counter_increment, tmp1);
3125     __ st_ptr(tmp1, counter_addr);
3126   }
3127 }
3128 
3129 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) {
3130   Register obj = op->obj()->as_register();
3131   Register tmp1 = op->tmp()->as_pointer_register();
3132   Register tmp2 = G1;
3133   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3134   ciKlass* exact_klass = op->exact_klass();
3135   intptr_t current_klass = op->current_klass();
3136   bool not_null = op->not_null();
3137   bool no_conflict = op->no_conflict();
3138 
3139   Label update, next, none;
3140 
3141   bool do_null = !not_null;
3142   bool exact_klass_set = exact_klass != NULL && ciTypeEntries::valid_ciklass(current_klass) == exact_klass;
3143   bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set;
3144 
3145   assert(do_null || do_update, "why are we here?");
3146   assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?");
3147 
3148   __ verify_oop(obj);
3149 
3150   if (tmp1 != obj) {
3151     __ mov(obj, tmp1);
3152   }
3153   if (do_null) {
3154     __ br_notnull_short(tmp1, Assembler::pt, update);
3155     if (!TypeEntries::was_null_seen(current_klass)) {
3156       __ ld_ptr(mdo_addr, tmp1);
3157       __ or3(tmp1, TypeEntries::null_seen, tmp1);
3158       __ st_ptr(tmp1, mdo_addr);
3159     }
3160     if (do_update) {
3161       __ ba(next);
3162       __ delayed()->nop();
3163     }
3164 #ifdef ASSERT
3165   } else {
3166     __ br_notnull_short(tmp1, Assembler::pt, update);
3167     __ stop("unexpect null obj");
3168 #endif
3169   }
3170 
3171   __ bind(update);
3172 
3173   if (do_update) {
3174 #ifdef ASSERT
3175     if (exact_klass != NULL) {
3176       Label ok;
3177       __ load_klass(tmp1, tmp1);
3178       metadata2reg(exact_klass->constant_encoding(), tmp2);
3179       __ cmp_and_br_short(tmp1, tmp2, Assembler::equal, Assembler::pt, ok);
3180       __ stop("exact klass and actual klass differ");
3181       __ bind(ok);
3182     }
3183 #endif
3184 
3185     Label do_update;
3186     __ ld_ptr(mdo_addr, tmp2);
3187 
3188     if (!no_conflict) {
3189       if (exact_klass == NULL || TypeEntries::is_type_none(current_klass)) {
3190         if (exact_klass != NULL) {
3191           metadata2reg(exact_klass->constant_encoding(), tmp1);
3192         } else {
3193           __ load_klass(tmp1, tmp1);
3194         }
3195 
3196         __ xor3(tmp1, tmp2, tmp1);
3197         __ btst(TypeEntries::type_klass_mask, tmp1);
3198         // klass seen before, nothing to do. The unknown bit may have been
3199         // set already but no need to check.
3200         __ brx(Assembler::zero, false, Assembler::pt, next);
3201         __ delayed()->
3202 
3203            btst(TypeEntries::type_unknown, tmp1);
3204         // already unknown. Nothing to do anymore.
3205         __ brx(Assembler::notZero, false, Assembler::pt, next);
3206 
3207         if (TypeEntries::is_type_none(current_klass)) {
3208           __ delayed()->btst(TypeEntries::type_mask, tmp2);
3209           __ brx(Assembler::zero, true, Assembler::pt, do_update);
3210           // first time here. Set profile type.
3211           __ delayed()->or3(tmp2, tmp1, tmp2);
3212         } else {
3213           __ delayed()->nop();
3214         }
3215       } else {
3216         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3217                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only");
3218 
3219         __ btst(TypeEntries::type_unknown, tmp2);
3220         // already unknown. Nothing to do anymore.
3221         __ brx(Assembler::notZero, false, Assembler::pt, next);
3222         __ delayed()->nop();
3223       }
3224 
3225       // different than before. Cannot keep accurate profile.
3226       __ or3(tmp2, TypeEntries::type_unknown, tmp2);
3227     } else {
3228       // There's a single possible klass at this profile point
3229       assert(exact_klass != NULL, "should be");
3230       if (TypeEntries::is_type_none(current_klass)) {
3231         metadata2reg(exact_klass->constant_encoding(), tmp1);
3232         __ xor3(tmp1, tmp2, tmp1);
3233         __ btst(TypeEntries::type_klass_mask, tmp1);
3234         __ brx(Assembler::zero, false, Assembler::pt, next);
3235 #ifdef ASSERT
3236 
3237         {
3238           Label ok;
3239           __ delayed()->btst(TypeEntries::type_mask, tmp2);
3240           __ brx(Assembler::zero, true, Assembler::pt, ok);
3241           __ delayed()->nop();
3242 
3243           __ stop("unexpected profiling mismatch");
3244           __ bind(ok);
3245         }
3246         // first time here. Set profile type.
3247         __ or3(tmp2, tmp1, tmp2);
3248 #else
3249         // first time here. Set profile type.
3250         __ delayed()->or3(tmp2, tmp1, tmp2);
3251 #endif
3252 
3253       } else {
3254         assert(ciTypeEntries::valid_ciklass(current_klass) != NULL &&
3255                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3256 
3257         // already unknown. Nothing to do anymore.
3258         __ btst(TypeEntries::type_unknown, tmp2);
3259         __ brx(Assembler::notZero, false, Assembler::pt, next);
3260         __ delayed()->or3(tmp2, TypeEntries::type_unknown, tmp2);
3261       }
3262     }
3263 
3264     __ bind(do_update);
3265     __ st_ptr(tmp2, mdo_addr);
3266 
3267     __ bind(next);
3268   }
3269 }
3270 
3271 void LIR_Assembler::align_backward_branch_target() {
3272   __ align(OptoLoopAlignment);
3273 }
3274 
3275 
3276 void LIR_Assembler::emit_delay(LIR_OpDelay* op) {
3277   // make sure we are expecting a delay
3278   // this has the side effect of clearing the delay state
3279   // so we can use _masm instead of _masm->delayed() to do the
3280   // code generation.
3281   __ delayed();
3282 
3283   // make sure we only emit one instruction
3284   int offset = code_offset();
3285   op->delay_op()->emit_code(this);
3286 #ifdef ASSERT
3287   if (code_offset() - offset != NativeInstruction::nop_instruction_size) {
3288     op->delay_op()->print();
3289   }
3290   assert(code_offset() - offset == NativeInstruction::nop_instruction_size,
3291          "only one instruction can go in a delay slot");
3292 #endif
3293 
3294   // we may also be emitting the call info for the instruction
3295   // which we are the delay slot of.
3296   CodeEmitInfo* call_info = op->call_info();
3297   if (call_info) {
3298     add_call_info(code_offset(), call_info);
3299   }
3300 
3301   if (VerifyStackAtCalls) {
3302     _masm->sub(FP, SP, O7);
3303     _masm->cmp(O7, initial_frame_size_in_bytes());
3304     _masm->trap(Assembler::notEqual, Assembler::ptr_cc, G0, ST_RESERVED_FOR_USER_0+2 );
3305   }
3306 }
3307 
3308 
3309 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest) {
3310   assert(left->is_register(), "can only handle registers");
3311 
3312   if (left->is_single_cpu()) {
3313     __ neg(left->as_register(), dest->as_register());
3314   } else if (left->is_single_fpu()) {
3315     __ fneg(FloatRegisterImpl::S, left->as_float_reg(), dest->as_float_reg());
3316   } else if (left->is_double_fpu()) {
3317     __ fneg(FloatRegisterImpl::D, left->as_double_reg(), dest->as_double_reg());
3318   } else {
3319     assert (left->is_double_cpu(), "Must be a long");
3320     Register Rlow = left->as_register_lo();
3321     Register Rhi = left->as_register_hi();
3322 #ifdef _LP64
3323     __ sub(G0, Rlow, dest->as_register_lo());
3324 #else
3325     __ subcc(G0, Rlow, dest->as_register_lo());
3326     __ subc (G0, Rhi,  dest->as_register_hi());
3327 #endif
3328   }
3329 }
3330 
3331 
3332 void LIR_Assembler::fxch(int i) {
3333   Unimplemented();
3334 }
3335 
3336 void LIR_Assembler::fld(int i) {
3337   Unimplemented();
3338 }
3339 
3340 void LIR_Assembler::ffree(int i) {
3341   Unimplemented();
3342 }
3343 
3344 void LIR_Assembler::rt_call(LIR_Opr result, address dest,
3345                             const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) {
3346 
3347   // if tmp is invalid, then the function being called doesn't destroy the thread
3348   if (tmp->is_valid()) {
3349     __ save_thread(tmp->as_register());
3350   }
3351   __ call(dest, relocInfo::runtime_call_type);
3352   __ delayed()->nop();
3353   if (info != NULL) {
3354     add_call_info_here(info);
3355   }
3356   if (tmp->is_valid()) {
3357     __ restore_thread(tmp->as_register());
3358   }
3359 
3360 #ifdef ASSERT
3361   __ verify_thread();
3362 #endif // ASSERT
3363 }
3364 
3365 
3366 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) {
3367 #ifdef _LP64
3368   ShouldNotReachHere();
3369 #endif
3370 
3371   NEEDS_CLEANUP;
3372   if (type == T_LONG) {
3373     LIR_Address* mem_addr = dest->is_address() ? dest->as_address_ptr() : src->as_address_ptr();
3374 
3375     // (extended to allow indexed as well as constant displaced for JSR-166)
3376     Register idx = noreg; // contains either constant offset or index
3377 
3378     int disp = mem_addr->disp();
3379     if (mem_addr->index() == LIR_OprFact::illegalOpr) {
3380       if (!Assembler::is_simm13(disp)) {
3381         idx = O7;
3382         __ set(disp, idx);
3383       }
3384     } else {
3385       assert(disp == 0, "not both indexed and disp");
3386       idx = mem_addr->index()->as_register();
3387     }
3388 
3389     int null_check_offset = -1;
3390 
3391     Register base = mem_addr->base()->as_register();
3392     if (src->is_register() && dest->is_address()) {
3393       // G4 is high half, G5 is low half
3394       // clear the top bits of G5, and scale up G4
3395       __ srl (src->as_register_lo(),  0, G5);
3396       __ sllx(src->as_register_hi(), 32, G4);
3397       // combine the two halves into the 64 bits of G4
3398       __ or3(G4, G5, G4);
3399       null_check_offset = __ offset();
3400       if (idx == noreg) {
3401         __ stx(G4, base, disp);
3402       } else {
3403         __ stx(G4, base, idx);
3404       }
3405     } else if (src->is_address() && dest->is_register()) {
3406       null_check_offset = __ offset();
3407       if (idx == noreg) {
3408         __ ldx(base, disp, G5);
3409       } else {
3410         __ ldx(base, idx, G5);
3411       }
3412       __ srax(G5, 32, dest->as_register_hi()); // fetch the high half into hi
3413       __ mov (G5, dest->as_register_lo());     // copy low half into lo
3414     } else {
3415       Unimplemented();
3416     }
3417     if (info != NULL) {
3418       add_debug_info_for_null_check(null_check_offset, info);
3419     }
3420 
3421   } else {
3422     // use normal move for all other volatiles since they don't need
3423     // special handling to remain atomic.
3424     move_op(src, dest, type, lir_patch_none, info, false, false, false);
3425   }
3426 }
3427 
3428 void LIR_Assembler::membar() {
3429   // only StoreLoad membars are ever explicitly needed on sparcs in TSO mode
3430   __ membar( Assembler::Membar_mask_bits(Assembler::StoreLoad) );
3431 }
3432 
3433 void LIR_Assembler::membar_acquire() {
3434   // no-op on TSO
3435 }
3436 
3437 void LIR_Assembler::membar_release() {
3438   // no-op on TSO
3439 }
3440 
3441 void LIR_Assembler::membar_loadload() {
3442   // no-op
3443   //__ membar(Assembler::Membar_mask_bits(Assembler::loadload));
3444 }
3445 
3446 void LIR_Assembler::membar_storestore() {
3447   // no-op
3448   //__ membar(Assembler::Membar_mask_bits(Assembler::storestore));
3449 }
3450 
3451 void LIR_Assembler::membar_loadstore() {
3452   // no-op
3453   //__ membar(Assembler::Membar_mask_bits(Assembler::loadstore));
3454 }
3455 
3456 void LIR_Assembler::membar_storeload() {
3457   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3458 }
3459 
3460 
3461 // Pack two sequential registers containing 32 bit values
3462 // into a single 64 bit register.
3463 // src and src->successor() are packed into dst
3464 // src and dst may be the same register.
3465 // Note: src is destroyed
3466 void LIR_Assembler::pack64(LIR_Opr src, LIR_Opr dst) {
3467   Register rs = src->as_register();
3468   Register rd = dst->as_register_lo();
3469   __ sllx(rs, 32, rs);
3470   __ srl(rs->successor(), 0, rs->successor());
3471   __ or3(rs, rs->successor(), rd);
3472 }
3473 
3474 // Unpack a 64 bit value in a register into
3475 // two sequential registers.
3476 // src is unpacked into dst and dst->successor()
3477 void LIR_Assembler::unpack64(LIR_Opr src, LIR_Opr dst) {
3478   Register rs = src->as_register_lo();
3479   Register rd = dst->as_register_hi();
3480   assert_different_registers(rs, rd, rd->successor());
3481   __ srlx(rs, 32, rd);
3482   __ srl (rs,  0, rd->successor());
3483 }
3484 
3485 
3486 void LIR_Assembler::leal(LIR_Opr addr_opr, LIR_Opr dest) {
3487   LIR_Address* addr = addr_opr->as_address_ptr();
3488   assert(addr->index()->is_illegal() && addr->scale() == LIR_Address::times_1, "can't handle complex addresses yet");
3489 
3490   if (Assembler::is_simm13(addr->disp())) {
3491     __ add(addr->base()->as_pointer_register(), addr->disp(), dest->as_pointer_register());
3492   } else {
3493     __ set(addr->disp(), G3_scratch);
3494     __ add(addr->base()->as_pointer_register(), G3_scratch, dest->as_pointer_register());
3495   }
3496 }
3497 
3498 
3499 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3500   assert(result_reg->is_register(), "check");
3501   __ mov(G2_thread, result_reg->as_register());
3502 }
3503 
3504 #ifdef ASSERT
3505 // emit run-time assertion
3506 void LIR_Assembler::emit_assert(LIR_OpAssert* op) {
3507   assert(op->code() == lir_assert, "must be");
3508 
3509   if (op->in_opr1()->is_valid()) {
3510     assert(op->in_opr2()->is_valid(), "both operands must be valid");
3511     comp_op(op->condition(), op->in_opr1(), op->in_opr2(), op);
3512   } else {
3513     assert(op->in_opr2()->is_illegal(), "both operands must be illegal");
3514     assert(op->condition() == lir_cond_always, "no other conditions allowed");
3515   }
3516 
3517   Label ok;
3518   if (op->condition() != lir_cond_always) {
3519     Assembler::Condition acond;
3520     switch (op->condition()) {
3521       case lir_cond_equal:        acond = Assembler::equal;                break;
3522       case lir_cond_notEqual:     acond = Assembler::notEqual;             break;
3523       case lir_cond_less:         acond = Assembler::less;                 break;
3524       case lir_cond_lessEqual:    acond = Assembler::lessEqual;            break;
3525       case lir_cond_greaterEqual: acond = Assembler::greaterEqual;         break;
3526       case lir_cond_greater:      acond = Assembler::greater;              break;
3527       case lir_cond_aboveEqual:   acond = Assembler::greaterEqualUnsigned; break;
3528       case lir_cond_belowEqual:   acond = Assembler::lessEqualUnsigned;    break;
3529       default:                         ShouldNotReachHere();
3530     };
3531     __ br(acond, false, Assembler::pt, ok);
3532     __ delayed()->nop();
3533   }
3534   if (op->halt()) {
3535     const char* str = __ code_string(op->msg());
3536     __ stop(str);
3537   } else {
3538     breakpoint();
3539   }
3540   __ bind(ok);
3541 }
3542 #endif
3543 
3544 void LIR_Assembler::peephole(LIR_List* lir) {
3545   LIR_OpList* inst = lir->instructions_list();
3546   for (int i = 0; i < inst->length(); i++) {
3547     LIR_Op* op = inst->at(i);
3548     switch (op->code()) {
3549       case lir_cond_float_branch:
3550       case lir_branch: {
3551         LIR_OpBranch* branch = op->as_OpBranch();
3552         assert(branch->info() == NULL, "shouldn't be state on branches anymore");
3553         LIR_Op* delay_op = NULL;
3554         // we'd like to be able to pull following instructions into
3555         // this slot but we don't know enough to do it safely yet so
3556         // only optimize block to block control flow.
3557         if (LIRFillDelaySlots && branch->block()) {
3558           LIR_Op* prev = inst->at(i - 1);
3559           if (prev && LIR_Assembler::is_single_instruction(prev) && prev->info() == NULL) {
3560             // swap previous instruction into delay slot
3561             inst->at_put(i - 1, op);
3562             inst->at_put(i, new LIR_OpDelay(prev, op->info()));
3563 #ifndef PRODUCT
3564             if (LIRTracePeephole) {
3565               tty->print_cr("delayed");
3566               inst->at(i - 1)->print();
3567               inst->at(i)->print();
3568               tty->cr();
3569             }
3570 #endif
3571             continue;
3572           }
3573         }
3574 
3575         if (!delay_op) {
3576           delay_op = new LIR_OpDelay(new LIR_Op0(lir_nop), NULL);
3577         }
3578         inst->insert_before(i + 1, delay_op);
3579         break;
3580       }
3581       case lir_static_call:
3582       case lir_virtual_call:
3583       case lir_icvirtual_call:
3584       case lir_optvirtual_call:
3585       case lir_dynamic_call: {
3586         LIR_Op* prev = inst->at(i - 1);
3587         if (LIRFillDelaySlots && prev && prev->code() == lir_move && prev->info() == NULL &&
3588             (op->code() != lir_virtual_call ||
3589              !prev->result_opr()->is_single_cpu() ||
3590              prev->result_opr()->as_register() != O0) &&
3591             LIR_Assembler::is_single_instruction(prev)) {
3592           // Only moves without info can be put into the delay slot.
3593           // Also don't allow the setup of the receiver in the delay
3594           // slot for vtable calls.
3595           inst->at_put(i - 1, op);
3596           inst->at_put(i, new LIR_OpDelay(prev, op->info()));
3597 #ifndef PRODUCT
3598           if (LIRTracePeephole) {
3599             tty->print_cr("delayed");
3600             inst->at(i - 1)->print();
3601             inst->at(i)->print();
3602             tty->cr();
3603           }
3604 #endif
3605         } else {
3606           LIR_Op* delay_op = new LIR_OpDelay(new LIR_Op0(lir_nop), op->as_OpJavaCall()->info());
3607           inst->insert_before(i + 1, delay_op);
3608           i++;
3609         }
3610 
3611 #if defined(TIERED) && !defined(_LP64)
3612         // fixup the return value from G1 to O0/O1 for long returns.
3613         // It's done here instead of in LIRGenerator because there's
3614         // such a mismatch between the single reg and double reg
3615         // calling convention.
3616         LIR_OpJavaCall* callop = op->as_OpJavaCall();
3617         if (callop->result_opr() == FrameMap::out_long_opr) {
3618           LIR_OpJavaCall* call;
3619           LIR_OprList* arguments = new LIR_OprList(callop->arguments()->length());
3620           for (int a = 0; a < arguments->length(); a++) {
3621             arguments[a] = callop->arguments()[a];
3622           }
3623           if (op->code() == lir_virtual_call) {
3624             call = new LIR_OpJavaCall(op->code(), callop->method(), callop->receiver(), FrameMap::g1_long_single_opr,
3625                                       callop->vtable_offset(), arguments, callop->info());
3626           } else {
3627             call = new LIR_OpJavaCall(op->code(), callop->method(), callop->receiver(), FrameMap::g1_long_single_opr,
3628                                       callop->addr(), arguments, callop->info());
3629           }
3630           inst->at_put(i - 1, call);
3631           inst->insert_before(i + 1, new LIR_Op1(lir_unpack64, FrameMap::g1_long_single_opr, callop->result_opr(),
3632                                                  T_LONG, lir_patch_none, NULL));
3633         }
3634 #endif
3635         break;
3636       }
3637     }
3638   }
3639 }
3640 
3641 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3642   LIR_Address* addr = src->as_address_ptr();
3643 
3644   assert(data == dest, "swap uses only 2 operands");
3645   assert (code == lir_xchg, "no xadd on sparc");
3646 
3647   if (data->type() == T_INT) {
3648     __ swap(as_Address(addr), data->as_register());
3649   } else if (data->is_oop()) {
3650     Register obj = data->as_register();
3651     Register narrow = tmp->as_register();
3652 #ifdef _LP64
3653     assert(UseCompressedOops, "swap is 32bit only");
3654     __ encode_heap_oop(obj, narrow);
3655     __ swap(as_Address(addr), narrow);
3656     __ decode_heap_oop(narrow, obj);
3657 #else
3658     __ swap(as_Address(addr), obj);
3659 #endif
3660   } else {
3661     ShouldNotReachHere();
3662   }
3663 }
3664 
3665 #undef __