< prev index next >

src/hotspot/cpu/x86/c1_LIRAssembler_x86.cpp

Print this page

  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"

  34 #include "ci/ciInstance.hpp"
  35 #include "compiler/oopMap.hpp"
  36 #include "gc/shared/collectedHeap.hpp"
  37 #include "gc/shared/gc_globals.hpp"
  38 #include "nativeInst_x86.hpp"

  39 #include "oops/objArrayKlass.hpp"
  40 #include "runtime/frame.inline.hpp"
  41 #include "runtime/safepointMechanism.hpp"
  42 #include "runtime/sharedRuntime.hpp"
  43 #include "runtime/stubRoutines.hpp"
  44 #include "utilities/powerOfTwo.hpp"
  45 #include "vmreg_x86.inline.hpp"
  46 
  47 
  48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  49 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  50 // fast versions of NegF/NegD and AbsF/AbsD.
  51 
  52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  54   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  55   // of 128-bits operands for SSE instructions.
  56   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  57   // Store the value to a 128-bits operand.
  58   operand[0] = lo;

 426     __ bind(*stub->continuation());
 427   }
 428 
 429   if (compilation()->env()->dtrace_method_probes()) {
 430 #ifdef _LP64
 431     __ mov(rdi, r15_thread);
 432     __ mov_metadata(rsi, method()->constant_encoding());
 433 #else
 434     __ get_thread(rax);
 435     __ movptr(Address(rsp, 0), rax);
 436     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
 437 #endif
 438     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 439   }
 440 
 441   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 442     __ mov(rax, rbx);  // Restore the exception
 443   }
 444 
 445   // remove the activation and dispatch to the unwind handler
 446   __ remove_frame(initial_frame_size_in_bytes());
 447   __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
 448 
 449   // Emit the slow path assembly
 450   if (stub != nullptr) {
 451     stub->emit_code(this);
 452   }
 453 
 454   return offset;
 455 }
 456 
 457 
 458 int LIR_Assembler::emit_deopt_handler() {
 459   // generate code for exception handler
 460   address handler_base = __ start_a_stub(deopt_handler_size());
 461   if (handler_base == nullptr) {
 462     // not enough space left for the handler
 463     bailout("deopt handler overflow");
 464     return -1;
 465   }
 466 
 467   int offset = code_offset();
 468   InternalAddress here(__ pc());
 469 
 470   __ pushptr(here.addr(), rscratch1);
 471   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 472   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 473   __ end_a_stub();
 474 
 475   return offset;
 476 }
 477 
 478 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 479   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 480   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 481     assert(result->fpu() == 0, "result must already be on TOS");
 482   }































 483 
 484   // Pop the stack before the safepoint code
 485   __ remove_frame(initial_frame_size_in_bytes());
 486 
 487   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 488     __ reserved_stack_check();
 489   }
 490 
 491   // Note: we do not need to round double result; float result has the right precision
 492   // the poll sets the condition code, but no data registers
 493 
 494 #ifdef _LP64
 495   const Register thread = r15_thread;
 496 #else
 497   const Register thread = rbx;
 498   __ get_thread(thread);
 499 #endif
 500   code_stub->set_safepoint_offset(__ offset());
 501   __ relocate(relocInfo::poll_return_type);
 502   __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
 503   __ ret(0);
 504 }
 505 
 506 




 507 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 508   guarantee(info != nullptr, "Shouldn't be null");
 509   int offset = __ offset();
 510 #ifdef _LP64
 511   const Register poll_addr = rscratch1;
 512   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 513 #else
 514   assert(tmp->is_cpu_register(), "needed");
 515   const Register poll_addr = tmp->as_register();
 516   __ get_thread(poll_addr);
 517   __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
 518 #endif
 519   add_debug_info_for_branch(info);
 520   __ relocate(relocInfo::poll_type);
 521   address pre_pc = __ pc();
 522   __ testl(rax, Address(poll_addr, 0));
 523   address post_pc = __ pc();
 524   guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
 525   return offset;
 526 }

1381     // init_state needs acquire, but x86 is TSO, and so we are already good.
1382     __ cmpb(Address(op->klass()->as_register(),
1383                     InstanceKlass::init_state_offset()),
1384                     InstanceKlass::fully_initialized);
1385     __ jcc(Assembler::notEqual, *op->stub()->entry());
1386   }
1387   __ allocate_object(op->obj()->as_register(),
1388                      op->tmp1()->as_register(),
1389                      op->tmp2()->as_register(),
1390                      op->header_size(),
1391                      op->object_size(),
1392                      op->klass()->as_register(),
1393                      *op->stub()->entry());
1394   __ bind(*op->stub()->continuation());
1395 }
1396 
1397 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1398   Register len =  op->len()->as_register();
1399   LP64_ONLY( __ movslq(len, len); )
1400 
1401   if (UseSlowPath ||
1402       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1403       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1404     __ jmp(*op->stub()->entry());
1405   } else {
1406     Register tmp1 = op->tmp1()->as_register();
1407     Register tmp2 = op->tmp2()->as_register();
1408     Register tmp3 = op->tmp3()->as_register();
1409     if (len == tmp1) {
1410       tmp1 = tmp3;
1411     } else if (len == tmp2) {
1412       tmp2 = tmp3;
1413     } else if (len == tmp3) {
1414       // everything is ok
1415     } else {
1416       __ mov(tmp3, len);
1417     }
1418     __ allocate_array(op->obj()->as_register(),
1419                       len,
1420                       tmp1,
1421                       tmp2,

1480     assert(data != nullptr,                "need data for type check");
1481     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1482   }
1483   Label* success_target = success;
1484   Label* failure_target = failure;
1485 
1486   if (obj == k_RInfo) {
1487     k_RInfo = dst;
1488   } else if (obj == klass_RInfo) {
1489     klass_RInfo = dst;
1490   }
1491   if (k->is_loaded() && !UseCompressedClassPointers) {
1492     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1493   } else {
1494     Rtmp1 = op->tmp3()->as_register();
1495     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1496   }
1497 
1498   assert_different_registers(obj, k_RInfo, klass_RInfo);
1499 
1500   __ testptr(obj, obj);
1501   if (op->should_profile()) {
1502     Label not_null;
1503     Register mdo  = klass_RInfo;
1504     __ mov_metadata(mdo, md->constant_encoding());
1505     __ jccb(Assembler::notEqual, not_null);
1506     // Object is null; update MDO and exit
1507     Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1508     int header_bits = BitData::null_seen_byte_constant();
1509     __ orb(data_addr, header_bits);
1510     __ jmp(*obj_is_null);
1511     __ bind(not_null);
1512 
1513     Label update_done;
1514     Register recv = k_RInfo;
1515     __ load_klass(recv, obj, tmp_load_klass);
1516     type_profile_helper(mdo, md, data, recv, &update_done);
1517 
1518     Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1519     __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1520 
1521     __ bind(update_done);
1522   } else {
1523     __ jcc(Assembler::equal, *obj_is_null);


1524   }
1525 
1526   if (!k->is_loaded()) {
1527     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1528   } else {
1529 #ifdef _LP64
1530     __ mov_metadata(k_RInfo, k->constant_encoding());
1531 #endif // _LP64
1532   }
1533   __ verify_oop(obj);
1534 
1535   if (op->fast_check()) {
1536     // get object class
1537     // not a safepoint as obj null check happens earlier
1538 #ifdef _LP64
1539     if (UseCompressedClassPointers) {
1540       __ load_klass(Rtmp1, obj, tmp_load_klass);
1541       __ cmpptr(k_RInfo, Rtmp1);
1542     } else {
1543       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));

1695         __ mov(dst, obj);
1696       }
1697     } else
1698       if (code == lir_instanceof) {
1699         Register obj = op->object()->as_register();
1700         Register dst = op->result_opr()->as_register();
1701         Label success, failure, done;
1702         emit_typecheck_helper(op, &success, &failure, &failure);
1703         __ bind(failure);
1704         __ xorptr(dst, dst);
1705         __ jmpb(done);
1706         __ bind(success);
1707         __ movptr(dst, 1);
1708         __ bind(done);
1709       } else {
1710         ShouldNotReachHere();
1711       }
1712 
1713 }
1714 







































































































1715 
1716 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1717   if (LP64_ONLY(false &&) op->code() == lir_cas_long) {
1718     assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1719     assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1720     assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1721     assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1722     Register addr = op->addr()->as_register();
1723     __ lock();
1724     NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1725 
1726   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1727     NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1728     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1729     Register newval = op->new_value()->as_register();
1730     Register cmpval = op->cmp_value()->as_register();
1731     assert(cmpval == rax, "wrong register");
1732     assert(newval != noreg, "new val must be register");
1733     assert(cmpval != newval, "cmp and new values must be in different registers");
1734     assert(cmpval != addr, "cmp and addr must be in different registers");

1755       __ cmpxchgl(newval, Address(addr, 0));
1756     }
1757 #ifdef _LP64
1758   } else if (op->code() == lir_cas_long) {
1759     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1760     Register newval = op->new_value()->as_register_lo();
1761     Register cmpval = op->cmp_value()->as_register_lo();
1762     assert(cmpval == rax, "wrong register");
1763     assert(newval != noreg, "new val must be register");
1764     assert(cmpval != newval, "cmp and new values must be in different registers");
1765     assert(cmpval != addr, "cmp and addr must be in different registers");
1766     assert(newval != addr, "new value and addr must be in different registers");
1767     __ lock();
1768     __ cmpxchgq(newval, Address(addr, 0));
1769 #endif // _LP64
1770   } else {
1771     Unimplemented();
1772   }
1773 }
1774 















1775 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1776                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1777   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1778 
1779   Assembler::Condition acond, ncond;
1780   switch (condition) {
1781     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1782     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1783     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1784     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1785     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1786     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1787     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1788     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1789     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1790                                 ShouldNotReachHere();
1791   }
1792 
1793   if (opr1->is_cpu_register()) {
1794     reg2reg(opr1, result);

2467   int offset = __ offset();
2468   switch (code) {
2469   case lir_static_call:
2470   case lir_optvirtual_call:
2471   case lir_dynamic_call:
2472     offset += NativeCall::displacement_offset;
2473     break;
2474   case lir_icvirtual_call:
2475     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2476     break;
2477   default: ShouldNotReachHere();
2478   }
2479   __ align(BytesPerWord, offset);
2480 }
2481 
2482 
2483 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2484   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2485          "must be aligned");
2486   __ call(AddressLiteral(op->addr(), rtype));
2487   add_call_info(code_offset(), op->info());
2488   __ post_call_nop();
2489 }
2490 
2491 
2492 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2493   __ ic_call(op->addr());
2494   add_call_info(code_offset(), op->info());
2495   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2496          "must be aligned");
2497   __ post_call_nop();
2498 }
2499 
2500 
2501 void LIR_Assembler::emit_static_call_stub() {
2502   address call_pc = __ pc();
2503   address stub = __ start_a_stub(call_stub_size());
2504   if (stub == nullptr) {
2505     bailout("static call stub overflow");
2506     return;
2507   }
2508 
2509   int start = __ offset();
2510 
2511   // make sure that the displacement word of the call ends up word aligned
2512   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2513   __ relocate(static_stub_Relocation::spec(call_pc));
2514   __ mov_metadata(rbx, (Metadata*)nullptr);

2655   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2656 }
2657 
2658 
2659 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2660   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2661   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2662   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2663   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2664 }
2665 
2666 
2667 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2668   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2669   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2670   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2671   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2672 }
2673 
2674 















2675 // This code replaces a call to arraycopy; no exception may
2676 // be thrown in this code, they must be thrown in the System.arraycopy
2677 // activation frame; we could save some checks if this would not be the case
2678 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2679   ciArrayKlass* default_type = op->expected_type();
2680   Register src = op->src()->as_register();
2681   Register dst = op->dst()->as_register();
2682   Register src_pos = op->src_pos()->as_register();
2683   Register dst_pos = op->dst_pos()->as_register();
2684   Register length  = op->length()->as_register();
2685   Register tmp = op->tmp()->as_register();
2686   Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
2687   Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2688 
2689   CodeStub* stub = op->stub();
2690   int flags = op->flags();
2691   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2692   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2693 






2694   // if we don't know anything, just go through the generic arraycopy
2695   if (default_type == nullptr) {
2696     // save outgoing arguments on stack in case call to System.arraycopy is needed
2697     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2698     // for interpreter calling conventions. Now we have to do it in new style conventions.
2699     // For the moment until C1 gets the new register allocator I just force all the
2700     // args to the right place (except the register args) and then on the back side
2701     // reload the register args properly if we go slow path. Yuck
2702 
2703     // These are proper for the calling convention
2704     store_parameter(length, 2);
2705     store_parameter(dst_pos, 1);
2706     store_parameter(dst, 0);
2707 
2708     // these are just temporary placements until we need to reload
2709     store_parameter(src_pos, 3);
2710     store_parameter(src, 4);
2711     NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
2712 
2713     address copyfunc_addr = StubRoutines::generic_arraycopy();

2767     __ mov(tmp, rax);
2768     __ xorl(tmp, -1);
2769 
2770     // Reload values from the stack so they are where the stub
2771     // expects them.
2772     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2773     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2774     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2775     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2776     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2777 
2778     __ subl(length, tmp);
2779     __ addl(src_pos, tmp);
2780     __ addl(dst_pos, tmp);
2781     __ jmp(*stub->entry());
2782 
2783     __ bind(*stub->continuation());
2784     return;
2785   }
2786 








2787   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2788 
2789   int elem_size = type2aelembytes(basic_type);
2790   Address::ScaleFactor scale;
2791 
2792   switch (elem_size) {
2793     case 1 :
2794       scale = Address::times_1;
2795       break;
2796     case 2 :
2797       scale = Address::times_2;
2798       break;
2799     case 4 :
2800       scale = Address::times_4;
2801       break;
2802     case 8 :
2803       scale = Address::times_8;
2804       break;
2805     default:
2806       scale = Address::no_scale;

3379         // first time here. Set profile type.
3380         __ movptr(mdo_addr, tmp);
3381 #ifdef ASSERT
3382         __ andptr(tmp, TypeEntries::type_klass_mask);
3383         __ verify_klass_ptr(tmp);
3384 #endif
3385       } else {
3386         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3387                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3388 
3389         __ testptr(mdo_addr, TypeEntries::type_unknown);
3390         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3391 
3392         __ orptr(mdo_addr, TypeEntries::type_unknown);
3393       }
3394     }
3395   }
3396   __ bind(next);
3397 }
3398 




















3399 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3400   Unimplemented();
3401 }
3402 
3403 
3404 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3405   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3406 }
3407 
3408 
3409 void LIR_Assembler::align_backward_branch_target() {
3410   __ align(BytesPerWord);
3411 }
3412 
3413 
3414 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3415   if (left->is_single_cpu()) {
3416     __ negl(left->as_register());
3417     move_regs(left->as_register(), dest->as_register());
3418 

3592 }
3593 
3594 void LIR_Assembler::membar_storeload() {
3595   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3596 }
3597 
3598 void LIR_Assembler::on_spin_wait() {
3599   __ pause ();
3600 }
3601 
3602 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3603   assert(result_reg->is_register(), "check");
3604 #ifdef _LP64
3605   // __ get_thread(result_reg->as_register_lo());
3606   __ mov(result_reg->as_register(), r15_thread);
3607 #else
3608   __ get_thread(result_reg->as_register());
3609 #endif // _LP64
3610 }
3611 



3612 
3613 void LIR_Assembler::peephole(LIR_List*) {
3614   // do nothing for now
3615 }
3616 
3617 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3618   assert(data == dest, "xchg/xadd uses only 2 operands");
3619 
3620   if (data->type() == T_INT) {
3621     if (code == lir_xadd) {
3622       __ lock();
3623       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3624     } else {
3625       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3626     }
3627   } else if (data->is_oop()) {
3628     assert (code == lir_xchg, "xadd for oops");
3629     Register obj = data->as_register();
3630 #ifdef _LP64
3631     if (UseCompressedOops) {

  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/macroAssembler.hpp"
  26 #include "asm/macroAssembler.inline.hpp"
  27 #include "c1/c1_CodeStubs.hpp"
  28 #include "c1/c1_Compilation.hpp"
  29 #include "c1/c1_LIRAssembler.hpp"
  30 #include "c1/c1_MacroAssembler.hpp"
  31 #include "c1/c1_Runtime1.hpp"
  32 #include "c1/c1_ValueStack.hpp"
  33 #include "ci/ciArrayKlass.hpp"
  34 #include "ci/ciInlineKlass.hpp"
  35 #include "ci/ciInstance.hpp"
  36 #include "compiler/oopMap.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "gc/shared/gc_globals.hpp"
  39 #include "nativeInst_x86.hpp"
  40 #include "oops/oop.inline.hpp"
  41 #include "oops/objArrayKlass.hpp"
  42 #include "runtime/frame.inline.hpp"
  43 #include "runtime/safepointMechanism.hpp"
  44 #include "runtime/sharedRuntime.hpp"
  45 #include "runtime/stubRoutines.hpp"
  46 #include "utilities/powerOfTwo.hpp"
  47 #include "vmreg_x86.inline.hpp"
  48 
  49 
  50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
  51 // instructions, to allow sign-masking or sign-bit flipping.  They allow
  52 // fast versions of NegF/NegD and AbsF/AbsD.
  53 
  54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
  55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
  56   // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
  57   // of 128-bits operands for SSE instructions.
  58   jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
  59   // Store the value to a 128-bits operand.
  60   operand[0] = lo;

 428     __ bind(*stub->continuation());
 429   }
 430 
 431   if (compilation()->env()->dtrace_method_probes()) {
 432 #ifdef _LP64
 433     __ mov(rdi, r15_thread);
 434     __ mov_metadata(rsi, method()->constant_encoding());
 435 #else
 436     __ get_thread(rax);
 437     __ movptr(Address(rsp, 0), rax);
 438     __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
 439 #endif
 440     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
 441   }
 442 
 443   if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
 444     __ mov(rax, rbx);  // Restore the exception
 445   }
 446 
 447   // remove the activation and dispatch to the unwind handler
 448   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 449   __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
 450 
 451   // Emit the slow path assembly
 452   if (stub != nullptr) {
 453     stub->emit_code(this);
 454   }
 455 
 456   return offset;
 457 }
 458 
 459 
 460 int LIR_Assembler::emit_deopt_handler() {
 461   // generate code for exception handler
 462   address handler_base = __ start_a_stub(deopt_handler_size());
 463   if (handler_base == nullptr) {
 464     // not enough space left for the handler
 465     bailout("deopt handler overflow");
 466     return -1;
 467   }
 468 
 469   int offset = code_offset();
 470   InternalAddress here(__ pc());
 471 
 472   __ pushptr(here.addr(), rscratch1);
 473   __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
 474   guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
 475   __ end_a_stub();
 476 
 477   return offset;
 478 }
 479 
 480 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
 481   assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
 482   if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
 483     assert(result->fpu() == 0, "result must already be on TOS");
 484   }
 485   if (InlineTypeReturnedAsFields) {
 486   #ifndef _LP64
 487      Unimplemented();
 488   #endif
 489     // Check if we are returning an non-null inline type and load its fields into registers
 490     ciType* return_type = compilation()->method()->return_type();
 491     if (return_type->is_inlinetype()) {
 492       ciInlineKlass* vk = return_type->as_inline_klass();
 493       if (vk->can_be_returned_as_fields()) {
 494         address unpack_handler = vk->unpack_handler();
 495         assert(unpack_handler != nullptr, "must be");
 496         __ call(RuntimeAddress(unpack_handler));
 497       }
 498     } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
 499       Label skip;
 500       __ test_oop_is_not_inline_type(rax, rscratch1, skip);
 501 
 502       // Load fields from a buffered value with an inline class specific handler
 503       __ load_klass(rdi, rax, rscratch1);
 504       __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
 505       __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
 506       // Unpack handler can be null if inline type is not scalarizable in returns
 507       __ testptr(rdi, rdi);
 508       __ jcc(Assembler::zero, skip);
 509       __ call(rdi);
 510 
 511       __ bind(skip);
 512     }
 513     // At this point, rax points to the value object (for interpreter or C1 caller).
 514     // The fields of the object are copied into registers (for C2 caller).
 515   }
 516 
 517   // Pop the stack before the safepoint code
 518   __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
 519 
 520   if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
 521     __ reserved_stack_check();
 522   }
 523 
 524   // Note: we do not need to round double result; float result has the right precision
 525   // the poll sets the condition code, but no data registers
 526 
 527 #ifdef _LP64
 528   const Register thread = r15_thread;
 529 #else
 530   const Register thread = rbx;
 531   __ get_thread(thread);
 532 #endif
 533   code_stub->set_safepoint_offset(__ offset());
 534   __ relocate(relocInfo::poll_return_type);
 535   __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
 536   __ ret(0);
 537 }
 538 
 539 
 540 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
 541   return (__ store_inline_type_fields_to_buf(vk, false));
 542 }
 543 
 544 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
 545   guarantee(info != nullptr, "Shouldn't be null");
 546   int offset = __ offset();
 547 #ifdef _LP64
 548   const Register poll_addr = rscratch1;
 549   __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
 550 #else
 551   assert(tmp->is_cpu_register(), "needed");
 552   const Register poll_addr = tmp->as_register();
 553   __ get_thread(poll_addr);
 554   __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
 555 #endif
 556   add_debug_info_for_branch(info);
 557   __ relocate(relocInfo::poll_type);
 558   address pre_pc = __ pc();
 559   __ testl(rax, Address(poll_addr, 0));
 560   address post_pc = __ pc();
 561   guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
 562   return offset;
 563 }

1418     // init_state needs acquire, but x86 is TSO, and so we are already good.
1419     __ cmpb(Address(op->klass()->as_register(),
1420                     InstanceKlass::init_state_offset()),
1421                     InstanceKlass::fully_initialized);
1422     __ jcc(Assembler::notEqual, *op->stub()->entry());
1423   }
1424   __ allocate_object(op->obj()->as_register(),
1425                      op->tmp1()->as_register(),
1426                      op->tmp2()->as_register(),
1427                      op->header_size(),
1428                      op->object_size(),
1429                      op->klass()->as_register(),
1430                      *op->stub()->entry());
1431   __ bind(*op->stub()->continuation());
1432 }
1433 
1434 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1435   Register len =  op->len()->as_register();
1436   LP64_ONLY( __ movslq(len, len); )
1437 
1438   if (UseSlowPath || op->is_null_free() ||
1439       (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1440       (!UseFastNewTypeArray   && !is_reference_type(op->type()))) {
1441     __ jmp(*op->stub()->entry());
1442   } else {
1443     Register tmp1 = op->tmp1()->as_register();
1444     Register tmp2 = op->tmp2()->as_register();
1445     Register tmp3 = op->tmp3()->as_register();
1446     if (len == tmp1) {
1447       tmp1 = tmp3;
1448     } else if (len == tmp2) {
1449       tmp2 = tmp3;
1450     } else if (len == tmp3) {
1451       // everything is ok
1452     } else {
1453       __ mov(tmp3, len);
1454     }
1455     __ allocate_array(op->obj()->as_register(),
1456                       len,
1457                       tmp1,
1458                       tmp2,

1517     assert(data != nullptr,                "need data for type check");
1518     assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1519   }
1520   Label* success_target = success;
1521   Label* failure_target = failure;
1522 
1523   if (obj == k_RInfo) {
1524     k_RInfo = dst;
1525   } else if (obj == klass_RInfo) {
1526     klass_RInfo = dst;
1527   }
1528   if (k->is_loaded() && !UseCompressedClassPointers) {
1529     select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1530   } else {
1531     Rtmp1 = op->tmp3()->as_register();
1532     select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1533   }
1534 
1535   assert_different_registers(obj, k_RInfo, klass_RInfo);
1536 
1537   if (op->need_null_check()) {
1538     __ testptr(obj, obj);
1539     if (op->should_profile()) {
1540       Label not_null;
1541       Register mdo  = klass_RInfo;
1542       __ mov_metadata(mdo, md->constant_encoding());
1543       __ jccb(Assembler::notEqual, not_null);
1544       // Object is null; update MDO and exit
1545       Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1546       int header_bits = BitData::null_seen_byte_constant();
1547       __ orb(data_addr, header_bits);
1548       __ jmp(*obj_is_null);
1549       __ bind(not_null);
1550 
1551       Label update_done;
1552       Register recv = k_RInfo;
1553       __ load_klass(recv, obj, tmp_load_klass);
1554       type_profile_helper(mdo, md, data, recv, &update_done);
1555 
1556       Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1557       __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1558 
1559       __ bind(update_done);
1560     } else {
1561       __ jcc(Assembler::equal, *obj_is_null);
1562     }
1563   }
1564 
1565   if (!k->is_loaded()) {
1566     klass2reg_with_patching(k_RInfo, op->info_for_patch());
1567   } else {
1568 #ifdef _LP64
1569     __ mov_metadata(k_RInfo, k->constant_encoding());
1570 #endif // _LP64
1571   }
1572   __ verify_oop(obj);
1573 
1574   if (op->fast_check()) {
1575     // get object class
1576     // not a safepoint as obj null check happens earlier
1577 #ifdef _LP64
1578     if (UseCompressedClassPointers) {
1579       __ load_klass(Rtmp1, obj, tmp_load_klass);
1580       __ cmpptr(k_RInfo, Rtmp1);
1581     } else {
1582       __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));

1734         __ mov(dst, obj);
1735       }
1736     } else
1737       if (code == lir_instanceof) {
1738         Register obj = op->object()->as_register();
1739         Register dst = op->result_opr()->as_register();
1740         Label success, failure, done;
1741         emit_typecheck_helper(op, &success, &failure, &failure);
1742         __ bind(failure);
1743         __ xorptr(dst, dst);
1744         __ jmpb(done);
1745         __ bind(success);
1746         __ movptr(dst, 1);
1747         __ bind(done);
1748       } else {
1749         ShouldNotReachHere();
1750       }
1751 
1752 }
1753 
1754 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1755   // We are loading/storing from/to an array that *may* be a flat array (the
1756   // declared type is Object[], abstract[], interface[] or VT.ref[]).
1757   // If this array is a flat array, take the slow path.
1758   __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1759   if (!op->value()->is_illegal()) {
1760     // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1761     // The array is not a flat array, but it might be null-free. If we are storing
1762     // a null into a null-free array, take the slow path (which will throw NPE).
1763     Label skip;
1764     __ cmpptr(op->value()->as_register(), NULL_WORD);
1765     __ jcc(Assembler::notEqual, skip);
1766     __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1767     __ bind(skip);
1768   }
1769 }
1770 
1771 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1772   // We are storing into an array that *may* be null-free (the declared type is
1773   // Object[], abstract[], interface[] or VT.ref[]).
1774   Label test_mark_word;
1775   Register tmp = op->tmp()->as_register();
1776   __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1777   __ testl(tmp, markWord::unlocked_value);
1778   __ jccb(Assembler::notZero, test_mark_word);
1779   __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1780   __ bind(test_mark_word);
1781   __ testl(tmp, markWord::null_free_array_bit_in_place);
1782 }
1783 
1784 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1785   Label L_oops_equal;
1786   Label L_oops_not_equal;
1787   Label L_end;
1788 
1789   Register left  = op->left()->as_register();
1790   Register right = op->right()->as_register();
1791 
1792   __ cmpptr(left, right);
1793   __ jcc(Assembler::equal, L_oops_equal);
1794 
1795   // (1) Null check -- if one of the operands is null, the other must not be null (because
1796   //     the two references are not equal), so they are not substitutable,
1797   //     FIXME: do null check only if the operand is nullable
1798   __ testptr(left, right);
1799   __ jcc(Assembler::zero, L_oops_not_equal);
1800 
1801   ciKlass* left_klass = op->left_klass();
1802   ciKlass* right_klass = op->right_klass();
1803 
1804   // (2) Inline type check -- if either of the operands is not a inline type,
1805   //     they are not substitutable. We do this only if we are not sure that the
1806   //     operands are inline type
1807   if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1808       !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1809     Register tmp1  = op->tmp1()->as_register();
1810     __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1811     __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1812     __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1813     __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1814     __ jcc(Assembler::notEqual, L_oops_not_equal);
1815   }
1816 
1817   // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1818   if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1819     // No need to load klass -- the operands are statically known to be the same inline klass.
1820     __ jmp(*op->stub()->entry());
1821   } else {
1822     Register left_klass_op = op->left_klass_op()->as_register();
1823     Register right_klass_op = op->right_klass_op()->as_register();
1824 
1825     if (UseCompressedClassPointers) {
1826       __ movl(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1827       __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1828       __ cmpl(left_klass_op, right_klass_op);
1829     } else {
1830       __ movptr(left_klass_op,  Address(left,  oopDesc::klass_offset_in_bytes()));
1831       __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1832       __ cmpptr(left_klass_op, right_klass_op);
1833     }
1834 
1835     __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1836     // fall through to L_oops_not_equal
1837   }
1838 
1839   __ bind(L_oops_not_equal);
1840   move(op->not_equal_result(), op->result_opr());
1841   __ jmp(L_end);
1842 
1843   __ bind(L_oops_equal);
1844   move(op->equal_result(), op->result_opr());
1845   __ jmp(L_end);
1846 
1847   // We've returned from the stub. RAX contains 0x0 IFF the two
1848   // operands are not substitutable. (Don't compare against 0x1 in case the
1849   // C compiler is naughty)
1850   __ bind(*op->stub()->continuation());
1851   __ cmpl(rax, 0);
1852   __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1853   move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1854   // fall-through
1855   __ bind(L_end);
1856 }
1857 
1858 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1859   if (LP64_ONLY(false &&) op->code() == lir_cas_long) {
1860     assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1861     assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1862     assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1863     assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1864     Register addr = op->addr()->as_register();
1865     __ lock();
1866     NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1867 
1868   } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1869     NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1870     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1871     Register newval = op->new_value()->as_register();
1872     Register cmpval = op->cmp_value()->as_register();
1873     assert(cmpval == rax, "wrong register");
1874     assert(newval != noreg, "new val must be register");
1875     assert(cmpval != newval, "cmp and new values must be in different registers");
1876     assert(cmpval != addr, "cmp and addr must be in different registers");

1897       __ cmpxchgl(newval, Address(addr, 0));
1898     }
1899 #ifdef _LP64
1900   } else if (op->code() == lir_cas_long) {
1901     Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1902     Register newval = op->new_value()->as_register_lo();
1903     Register cmpval = op->cmp_value()->as_register_lo();
1904     assert(cmpval == rax, "wrong register");
1905     assert(newval != noreg, "new val must be register");
1906     assert(cmpval != newval, "cmp and new values must be in different registers");
1907     assert(cmpval != addr, "cmp and addr must be in different registers");
1908     assert(newval != addr, "new value and addr must be in different registers");
1909     __ lock();
1910     __ cmpxchgq(newval, Address(addr, 0));
1911 #endif // _LP64
1912   } else {
1913     Unimplemented();
1914   }
1915 }
1916 
1917 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1918   assert(dst->is_cpu_register(), "must be");
1919   assert(dst->type() == src->type(), "must be");
1920 
1921   if (src->is_cpu_register()) {
1922     reg2reg(src, dst);
1923   } else if (src->is_stack()) {
1924     stack2reg(src, dst, dst->type());
1925   } else if (src->is_constant()) {
1926     const2reg(src, dst, lir_patch_none, nullptr);
1927   } else {
1928     ShouldNotReachHere();
1929   }
1930 }
1931 
1932 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1933                           LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1934   assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1935 
1936   Assembler::Condition acond, ncond;
1937   switch (condition) {
1938     case lir_cond_equal:        acond = Assembler::equal;        ncond = Assembler::notEqual;     break;
1939     case lir_cond_notEqual:     acond = Assembler::notEqual;     ncond = Assembler::equal;        break;
1940     case lir_cond_less:         acond = Assembler::less;         ncond = Assembler::greaterEqual; break;
1941     case lir_cond_lessEqual:    acond = Assembler::lessEqual;    ncond = Assembler::greater;      break;
1942     case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less;         break;
1943     case lir_cond_greater:      acond = Assembler::greater;      ncond = Assembler::lessEqual;    break;
1944     case lir_cond_belowEqual:   acond = Assembler::belowEqual;   ncond = Assembler::above;        break;
1945     case lir_cond_aboveEqual:   acond = Assembler::aboveEqual;   ncond = Assembler::below;        break;
1946     default:                    acond = Assembler::equal;        ncond = Assembler::notEqual;
1947                                 ShouldNotReachHere();
1948   }
1949 
1950   if (opr1->is_cpu_register()) {
1951     reg2reg(opr1, result);

2624   int offset = __ offset();
2625   switch (code) {
2626   case lir_static_call:
2627   case lir_optvirtual_call:
2628   case lir_dynamic_call:
2629     offset += NativeCall::displacement_offset;
2630     break;
2631   case lir_icvirtual_call:
2632     offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2633     break;
2634   default: ShouldNotReachHere();
2635   }
2636   __ align(BytesPerWord, offset);
2637 }
2638 
2639 
2640 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2641   assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2642          "must be aligned");
2643   __ call(AddressLiteral(op->addr(), rtype));
2644   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2645   __ post_call_nop();
2646 }
2647 
2648 
2649 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2650   __ ic_call(op->addr());
2651   add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2652   assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2653          "must be aligned");
2654   __ post_call_nop();
2655 }
2656 
2657 
2658 void LIR_Assembler::emit_static_call_stub() {
2659   address call_pc = __ pc();
2660   address stub = __ start_a_stub(call_stub_size());
2661   if (stub == nullptr) {
2662     bailout("static call stub overflow");
2663     return;
2664   }
2665 
2666   int start = __ offset();
2667 
2668   // make sure that the displacement word of the call ends up word aligned
2669   __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2670   __ relocate(static_stub_Relocation::spec(call_pc));
2671   __ mov_metadata(rbx, (Metadata*)nullptr);

2812   __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2813 }
2814 
2815 
2816 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2817   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2818   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2819   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2820   __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2821 }
2822 
2823 
2824 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2825   assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2826   int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2827   assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2828   __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2829 }
2830 
2831 
2832 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2833   if (null_check) {
2834     __ testptr(obj, obj);
2835     __ jcc(Assembler::zero, *slow_path->entry());
2836   }
2837   if (is_dest) {
2838     __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2839     // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2840     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2841   } else {
2842     __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2843   }
2844 }
2845 
2846 
2847 // This code replaces a call to arraycopy; no exception may
2848 // be thrown in this code, they must be thrown in the System.arraycopy
2849 // activation frame; we could save some checks if this would not be the case
2850 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2851   ciArrayKlass* default_type = op->expected_type();
2852   Register src = op->src()->as_register();
2853   Register dst = op->dst()->as_register();
2854   Register src_pos = op->src_pos()->as_register();
2855   Register dst_pos = op->dst_pos()->as_register();
2856   Register length  = op->length()->as_register();
2857   Register tmp = op->tmp()->as_register();
2858   Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
2859   Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2860 
2861   CodeStub* stub = op->stub();
2862   int flags = op->flags();
2863   BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2864   if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2865 
2866   if (flags & LIR_OpArrayCopy::always_slow_path) {
2867     __ jmp(*stub->entry());
2868     __ bind(*stub->continuation());
2869     return;
2870   }
2871 
2872   // if we don't know anything, just go through the generic arraycopy
2873   if (default_type == nullptr) {
2874     // save outgoing arguments on stack in case call to System.arraycopy is needed
2875     // HACK ALERT. This code used to push the parameters in a hardwired fashion
2876     // for interpreter calling conventions. Now we have to do it in new style conventions.
2877     // For the moment until C1 gets the new register allocator I just force all the
2878     // args to the right place (except the register args) and then on the back side
2879     // reload the register args properly if we go slow path. Yuck
2880 
2881     // These are proper for the calling convention
2882     store_parameter(length, 2);
2883     store_parameter(dst_pos, 1);
2884     store_parameter(dst, 0);
2885 
2886     // these are just temporary placements until we need to reload
2887     store_parameter(src_pos, 3);
2888     store_parameter(src, 4);
2889     NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
2890 
2891     address copyfunc_addr = StubRoutines::generic_arraycopy();

2945     __ mov(tmp, rax);
2946     __ xorl(tmp, -1);
2947 
2948     // Reload values from the stack so they are where the stub
2949     // expects them.
2950     __ movptr   (dst,     Address(rsp, 0*BytesPerWord));
2951     __ movptr   (dst_pos, Address(rsp, 1*BytesPerWord));
2952     __ movptr   (length,  Address(rsp, 2*BytesPerWord));
2953     __ movptr   (src_pos, Address(rsp, 3*BytesPerWord));
2954     __ movptr   (src,     Address(rsp, 4*BytesPerWord));
2955 
2956     __ subl(length, tmp);
2957     __ addl(src_pos, tmp);
2958     __ addl(dst_pos, tmp);
2959     __ jmp(*stub->entry());
2960 
2961     __ bind(*stub->continuation());
2962     return;
2963   }
2964 
2965   // Handle inline type arrays
2966   if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2967     arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2968   }
2969   if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2970     arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2971   }
2972 
2973   assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2974 
2975   int elem_size = type2aelembytes(basic_type);
2976   Address::ScaleFactor scale;
2977 
2978   switch (elem_size) {
2979     case 1 :
2980       scale = Address::times_1;
2981       break;
2982     case 2 :
2983       scale = Address::times_2;
2984       break;
2985     case 4 :
2986       scale = Address::times_4;
2987       break;
2988     case 8 :
2989       scale = Address::times_8;
2990       break;
2991     default:
2992       scale = Address::no_scale;

3565         // first time here. Set profile type.
3566         __ movptr(mdo_addr, tmp);
3567 #ifdef ASSERT
3568         __ andptr(tmp, TypeEntries::type_klass_mask);
3569         __ verify_klass_ptr(tmp);
3570 #endif
3571       } else {
3572         assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3573                ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3574 
3575         __ testptr(mdo_addr, TypeEntries::type_unknown);
3576         __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3577 
3578         __ orptr(mdo_addr, TypeEntries::type_unknown);
3579       }
3580     }
3581   }
3582   __ bind(next);
3583 }
3584 
3585 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3586   Register obj = op->obj()->as_register();
3587   Register tmp = op->tmp()->as_pointer_register();
3588   Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3589   bool not_null = op->not_null();
3590   int flag = op->flag();
3591 
3592   Label not_inline_type;
3593   if (!not_null) {
3594     __ testptr(obj, obj);
3595     __ jccb(Assembler::zero, not_inline_type);
3596   }
3597 
3598   __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3599 
3600   __ orb(mdo_addr, flag);
3601 
3602   __ bind(not_inline_type);
3603 }
3604 
3605 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3606   Unimplemented();
3607 }
3608 
3609 
3610 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3611   __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3612 }
3613 
3614 
3615 void LIR_Assembler::align_backward_branch_target() {
3616   __ align(BytesPerWord);
3617 }
3618 
3619 
3620 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3621   if (left->is_single_cpu()) {
3622     __ negl(left->as_register());
3623     move_regs(left->as_register(), dest->as_register());
3624 

3798 }
3799 
3800 void LIR_Assembler::membar_storeload() {
3801   __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3802 }
3803 
3804 void LIR_Assembler::on_spin_wait() {
3805   __ pause ();
3806 }
3807 
3808 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3809   assert(result_reg->is_register(), "check");
3810 #ifdef _LP64
3811   // __ get_thread(result_reg->as_register_lo());
3812   __ mov(result_reg->as_register(), r15_thread);
3813 #else
3814   __ get_thread(result_reg->as_register());
3815 #endif // _LP64
3816 }
3817 
3818 void LIR_Assembler::check_orig_pc() {
3819   __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3820 }
3821 
3822 void LIR_Assembler::peephole(LIR_List*) {
3823   // do nothing for now
3824 }
3825 
3826 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3827   assert(data == dest, "xchg/xadd uses only 2 operands");
3828 
3829   if (data->type() == T_INT) {
3830     if (code == lir_xadd) {
3831       __ lock();
3832       __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3833     } else {
3834       __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3835     }
3836   } else if (data->is_oop()) {
3837     assert (code == lir_xchg, "xadd for oops");
3838     Register obj = data->as_register();
3839 #ifdef _LP64
3840     if (UseCompressedOops) {
< prev index next >