14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInstance.hpp"
35 #include "compiler/oopMap.hpp"
36 #include "gc/shared/collectedHeap.hpp"
37 #include "gc/shared/gc_globals.hpp"
38 #include "nativeInst_x86.hpp"
39 #include "oops/objArrayKlass.hpp"
40 #include "runtime/frame.inline.hpp"
41 #include "runtime/safepointMechanism.hpp"
42 #include "runtime/sharedRuntime.hpp"
43 #include "runtime/stubRoutines.hpp"
44 #include "utilities/powerOfTwo.hpp"
45 #include "vmreg_x86.inline.hpp"
46
47
48 // These masks are used to provide 128-bit aligned bitmasks to the XMM
49 // instructions, to allow sign-masking or sign-bit flipping. They allow
50 // fast versions of NegF/NegD and AbsF/AbsD.
51
52 // Note: 'double' and 'long long' have 32-bits alignment on x86.
53 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
54 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
55 // of 128-bits operands for SSE instructions.
56 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
57 // Store the value to a 128-bits operand.
58 operand[0] = lo;
426 __ bind(*stub->continuation());
427 }
428
429 if (compilation()->env()->dtrace_method_probes()) {
430 #ifdef _LP64
431 __ mov(rdi, r15_thread);
432 __ mov_metadata(rsi, method()->constant_encoding());
433 #else
434 __ get_thread(rax);
435 __ movptr(Address(rsp, 0), rax);
436 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
437 #endif
438 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
439 }
440
441 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
442 __ mov(rax, rbx); // Restore the exception
443 }
444
445 // remove the activation and dispatch to the unwind handler
446 __ remove_frame(initial_frame_size_in_bytes());
447 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
448
449 // Emit the slow path assembly
450 if (stub != nullptr) {
451 stub->emit_code(this);
452 }
453
454 return offset;
455 }
456
457
458 int LIR_Assembler::emit_deopt_handler() {
459 // generate code for exception handler
460 address handler_base = __ start_a_stub(deopt_handler_size());
461 if (handler_base == nullptr) {
462 // not enough space left for the handler
463 bailout("deopt handler overflow");
464 return -1;
465 }
466
467 int offset = code_offset();
468 InternalAddress here(__ pc());
469
470 __ pushptr(here.addr(), rscratch1);
471 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
472 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
473 __ end_a_stub();
474
475 return offset;
476 }
477
478 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
479 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
480 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
481 assert(result->fpu() == 0, "result must already be on TOS");
482 }
483
484 // Pop the stack before the safepoint code
485 __ remove_frame(initial_frame_size_in_bytes());
486
487 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
488 __ reserved_stack_check();
489 }
490
491 // Note: we do not need to round double result; float result has the right precision
492 // the poll sets the condition code, but no data registers
493
494 #ifdef _LP64
495 const Register thread = r15_thread;
496 #else
497 const Register thread = rbx;
498 __ get_thread(thread);
499 #endif
500 code_stub->set_safepoint_offset(__ offset());
501 __ relocate(relocInfo::poll_return_type);
502 __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
503 __ ret(0);
504 }
505
506
507 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
508 guarantee(info != nullptr, "Shouldn't be null");
509 int offset = __ offset();
510 #ifdef _LP64
511 const Register poll_addr = rscratch1;
512 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
513 #else
514 assert(tmp->is_cpu_register(), "needed");
515 const Register poll_addr = tmp->as_register();
516 __ get_thread(poll_addr);
517 __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
518 #endif
519 add_debug_info_for_branch(info);
520 __ relocate(relocInfo::poll_type);
521 address pre_pc = __ pc();
522 __ testl(rax, Address(poll_addr, 0));
523 address post_pc = __ pc();
524 guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
525 return offset;
526 }
1381 // init_state needs acquire, but x86 is TSO, and so we are already good.
1382 __ cmpb(Address(op->klass()->as_register(),
1383 InstanceKlass::init_state_offset()),
1384 InstanceKlass::fully_initialized);
1385 __ jcc(Assembler::notEqual, *op->stub()->entry());
1386 }
1387 __ allocate_object(op->obj()->as_register(),
1388 op->tmp1()->as_register(),
1389 op->tmp2()->as_register(),
1390 op->header_size(),
1391 op->object_size(),
1392 op->klass()->as_register(),
1393 *op->stub()->entry());
1394 __ bind(*op->stub()->continuation());
1395 }
1396
1397 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1398 Register len = op->len()->as_register();
1399 LP64_ONLY( __ movslq(len, len); )
1400
1401 if (UseSlowPath ||
1402 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1403 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1404 __ jmp(*op->stub()->entry());
1405 } else {
1406 Register tmp1 = op->tmp1()->as_register();
1407 Register tmp2 = op->tmp2()->as_register();
1408 Register tmp3 = op->tmp3()->as_register();
1409 if (len == tmp1) {
1410 tmp1 = tmp3;
1411 } else if (len == tmp2) {
1412 tmp2 = tmp3;
1413 } else if (len == tmp3) {
1414 // everything is ok
1415 } else {
1416 __ mov(tmp3, len);
1417 }
1418 __ allocate_array(op->obj()->as_register(),
1419 len,
1420 tmp1,
1421 tmp2,
1480 assert(data != nullptr, "need data for type check");
1481 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1482 }
1483 Label* success_target = success;
1484 Label* failure_target = failure;
1485
1486 if (obj == k_RInfo) {
1487 k_RInfo = dst;
1488 } else if (obj == klass_RInfo) {
1489 klass_RInfo = dst;
1490 }
1491 if (k->is_loaded() && !UseCompressedClassPointers) {
1492 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1493 } else {
1494 Rtmp1 = op->tmp3()->as_register();
1495 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1496 }
1497
1498 assert_different_registers(obj, k_RInfo, klass_RInfo);
1499
1500 __ testptr(obj, obj);
1501 if (op->should_profile()) {
1502 Label not_null;
1503 Register mdo = klass_RInfo;
1504 __ mov_metadata(mdo, md->constant_encoding());
1505 __ jccb(Assembler::notEqual, not_null);
1506 // Object is null; update MDO and exit
1507 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1508 int header_bits = BitData::null_seen_byte_constant();
1509 __ orb(data_addr, header_bits);
1510 __ jmp(*obj_is_null);
1511 __ bind(not_null);
1512
1513 Label update_done;
1514 Register recv = k_RInfo;
1515 __ load_klass(recv, obj, tmp_load_klass);
1516 type_profile_helper(mdo, md, data, recv, &update_done);
1517
1518 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1519 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1520
1521 __ bind(update_done);
1522 } else {
1523 __ jcc(Assembler::equal, *obj_is_null);
1524 }
1525
1526 if (!k->is_loaded()) {
1527 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1528 } else {
1529 #ifdef _LP64
1530 __ mov_metadata(k_RInfo, k->constant_encoding());
1531 #endif // _LP64
1532 }
1533 __ verify_oop(obj);
1534
1535 if (op->fast_check()) {
1536 // get object class
1537 // not a safepoint as obj null check happens earlier
1538 #ifdef _LP64
1539 if (UseCompressedClassPointers) {
1540 __ load_klass(Rtmp1, obj, tmp_load_klass);
1541 __ cmpptr(k_RInfo, Rtmp1);
1542 } else {
1543 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1695 __ mov(dst, obj);
1696 }
1697 } else
1698 if (code == lir_instanceof) {
1699 Register obj = op->object()->as_register();
1700 Register dst = op->result_opr()->as_register();
1701 Label success, failure, done;
1702 emit_typecheck_helper(op, &success, &failure, &failure);
1703 __ bind(failure);
1704 __ xorptr(dst, dst);
1705 __ jmpb(done);
1706 __ bind(success);
1707 __ movptr(dst, 1);
1708 __ bind(done);
1709 } else {
1710 ShouldNotReachHere();
1711 }
1712
1713 }
1714
1715
1716 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1717 if (LP64_ONLY(false &&) op->code() == lir_cas_long) {
1718 assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1719 assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1720 assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1721 assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1722 Register addr = op->addr()->as_register();
1723 __ lock();
1724 NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1725
1726 } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1727 NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1728 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1729 Register newval = op->new_value()->as_register();
1730 Register cmpval = op->cmp_value()->as_register();
1731 assert(cmpval == rax, "wrong register");
1732 assert(newval != noreg, "new val must be register");
1733 assert(cmpval != newval, "cmp and new values must be in different registers");
1734 assert(cmpval != addr, "cmp and addr must be in different registers");
1755 __ cmpxchgl(newval, Address(addr, 0));
1756 }
1757 #ifdef _LP64
1758 } else if (op->code() == lir_cas_long) {
1759 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1760 Register newval = op->new_value()->as_register_lo();
1761 Register cmpval = op->cmp_value()->as_register_lo();
1762 assert(cmpval == rax, "wrong register");
1763 assert(newval != noreg, "new val must be register");
1764 assert(cmpval != newval, "cmp and new values must be in different registers");
1765 assert(cmpval != addr, "cmp and addr must be in different registers");
1766 assert(newval != addr, "new value and addr must be in different registers");
1767 __ lock();
1768 __ cmpxchgq(newval, Address(addr, 0));
1769 #endif // _LP64
1770 } else {
1771 Unimplemented();
1772 }
1773 }
1774
1775 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1776 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1777 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1778
1779 Assembler::Condition acond, ncond;
1780 switch (condition) {
1781 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1782 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1783 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1784 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1785 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1786 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1787 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1788 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1789 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1790 ShouldNotReachHere();
1791 }
1792
1793 if (opr1->is_cpu_register()) {
1794 reg2reg(opr1, result);
2467 int offset = __ offset();
2468 switch (code) {
2469 case lir_static_call:
2470 case lir_optvirtual_call:
2471 case lir_dynamic_call:
2472 offset += NativeCall::displacement_offset;
2473 break;
2474 case lir_icvirtual_call:
2475 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2476 break;
2477 default: ShouldNotReachHere();
2478 }
2479 __ align(BytesPerWord, offset);
2480 }
2481
2482
2483 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2484 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2485 "must be aligned");
2486 __ call(AddressLiteral(op->addr(), rtype));
2487 add_call_info(code_offset(), op->info());
2488 __ post_call_nop();
2489 }
2490
2491
2492 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2493 __ ic_call(op->addr());
2494 add_call_info(code_offset(), op->info());
2495 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2496 "must be aligned");
2497 __ post_call_nop();
2498 }
2499
2500
2501 void LIR_Assembler::emit_static_call_stub() {
2502 address call_pc = __ pc();
2503 address stub = __ start_a_stub(call_stub_size());
2504 if (stub == nullptr) {
2505 bailout("static call stub overflow");
2506 return;
2507 }
2508
2509 int start = __ offset();
2510
2511 // make sure that the displacement word of the call ends up word aligned
2512 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2513 __ relocate(static_stub_Relocation::spec(call_pc));
2514 __ mov_metadata(rbx, (Metadata*)nullptr);
2655 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2656 }
2657
2658
2659 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2660 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2661 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2662 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2663 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2664 }
2665
2666
2667 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2668 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2669 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2670 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2671 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2672 }
2673
2674
2675 // This code replaces a call to arraycopy; no exception may
2676 // be thrown in this code, they must be thrown in the System.arraycopy
2677 // activation frame; we could save some checks if this would not be the case
2678 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2679 ciArrayKlass* default_type = op->expected_type();
2680 Register src = op->src()->as_register();
2681 Register dst = op->dst()->as_register();
2682 Register src_pos = op->src_pos()->as_register();
2683 Register dst_pos = op->dst_pos()->as_register();
2684 Register length = op->length()->as_register();
2685 Register tmp = op->tmp()->as_register();
2686 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
2687 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2688
2689 CodeStub* stub = op->stub();
2690 int flags = op->flags();
2691 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2692 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2693
2694 // if we don't know anything, just go through the generic arraycopy
2695 if (default_type == nullptr) {
2696 // save outgoing arguments on stack in case call to System.arraycopy is needed
2697 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2698 // for interpreter calling conventions. Now we have to do it in new style conventions.
2699 // For the moment until C1 gets the new register allocator I just force all the
2700 // args to the right place (except the register args) and then on the back side
2701 // reload the register args properly if we go slow path. Yuck
2702
2703 // These are proper for the calling convention
2704 store_parameter(length, 2);
2705 store_parameter(dst_pos, 1);
2706 store_parameter(dst, 0);
2707
2708 // these are just temporary placements until we need to reload
2709 store_parameter(src_pos, 3);
2710 store_parameter(src, 4);
2711 NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
2712
2713 address copyfunc_addr = StubRoutines::generic_arraycopy();
2767 __ mov(tmp, rax);
2768 __ xorl(tmp, -1);
2769
2770 // Reload values from the stack so they are where the stub
2771 // expects them.
2772 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2773 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2774 __ movptr (length, Address(rsp, 2*BytesPerWord));
2775 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2776 __ movptr (src, Address(rsp, 4*BytesPerWord));
2777
2778 __ subl(length, tmp);
2779 __ addl(src_pos, tmp);
2780 __ addl(dst_pos, tmp);
2781 __ jmp(*stub->entry());
2782
2783 __ bind(*stub->continuation());
2784 return;
2785 }
2786
2787 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2788
2789 int elem_size = type2aelembytes(basic_type);
2790 Address::ScaleFactor scale;
2791
2792 switch (elem_size) {
2793 case 1 :
2794 scale = Address::times_1;
2795 break;
2796 case 2 :
2797 scale = Address::times_2;
2798 break;
2799 case 4 :
2800 scale = Address::times_4;
2801 break;
2802 case 8 :
2803 scale = Address::times_8;
2804 break;
2805 default:
2806 scale = Address::no_scale;
3379 // first time here. Set profile type.
3380 __ movptr(mdo_addr, tmp);
3381 #ifdef ASSERT
3382 __ andptr(tmp, TypeEntries::type_klass_mask);
3383 __ verify_klass_ptr(tmp);
3384 #endif
3385 } else {
3386 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3387 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3388
3389 __ testptr(mdo_addr, TypeEntries::type_unknown);
3390 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3391
3392 __ orptr(mdo_addr, TypeEntries::type_unknown);
3393 }
3394 }
3395 }
3396 __ bind(next);
3397 }
3398
3399 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3400 Unimplemented();
3401 }
3402
3403
3404 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3405 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3406 }
3407
3408
3409 void LIR_Assembler::align_backward_branch_target() {
3410 __ align(BytesPerWord);
3411 }
3412
3413
3414 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3415 if (left->is_single_cpu()) {
3416 __ negl(left->as_register());
3417 move_regs(left->as_register(), dest->as_register());
3418
3592 }
3593
3594 void LIR_Assembler::membar_storeload() {
3595 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3596 }
3597
3598 void LIR_Assembler::on_spin_wait() {
3599 __ pause ();
3600 }
3601
3602 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3603 assert(result_reg->is_register(), "check");
3604 #ifdef _LP64
3605 // __ get_thread(result_reg->as_register_lo());
3606 __ mov(result_reg->as_register(), r15_thread);
3607 #else
3608 __ get_thread(result_reg->as_register());
3609 #endif // _LP64
3610 }
3611
3612
3613 void LIR_Assembler::peephole(LIR_List*) {
3614 // do nothing for now
3615 }
3616
3617 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3618 assert(data == dest, "xchg/xadd uses only 2 operands");
3619
3620 if (data->type() == T_INT) {
3621 if (code == lir_xadd) {
3622 __ lock();
3623 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3624 } else {
3625 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3626 }
3627 } else if (data->is_oop()) {
3628 assert (code == lir_xchg, "xadd for oops");
3629 Register obj = data->as_register();
3630 #ifdef _LP64
3631 if (UseCompressedOops) {
|
14 *
15 * You should have received a copy of the GNU General Public License version
16 * 2 along with this work; if not, write to the Free Software Foundation,
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
18 *
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
20 * or visit www.oracle.com if you need additional information or have any
21 * questions.
22 *
23 */
24
25 #include "asm/macroAssembler.hpp"
26 #include "asm/macroAssembler.inline.hpp"
27 #include "c1/c1_CodeStubs.hpp"
28 #include "c1/c1_Compilation.hpp"
29 #include "c1/c1_LIRAssembler.hpp"
30 #include "c1/c1_MacroAssembler.hpp"
31 #include "c1/c1_Runtime1.hpp"
32 #include "c1/c1_ValueStack.hpp"
33 #include "ci/ciArrayKlass.hpp"
34 #include "ci/ciInlineKlass.hpp"
35 #include "ci/ciInstance.hpp"
36 #include "compiler/oopMap.hpp"
37 #include "gc/shared/collectedHeap.hpp"
38 #include "gc/shared/gc_globals.hpp"
39 #include "nativeInst_x86.hpp"
40 #include "oops/oop.inline.hpp"
41 #include "oops/objArrayKlass.hpp"
42 #include "runtime/frame.inline.hpp"
43 #include "runtime/safepointMechanism.hpp"
44 #include "runtime/sharedRuntime.hpp"
45 #include "runtime/stubRoutines.hpp"
46 #include "utilities/powerOfTwo.hpp"
47 #include "vmreg_x86.inline.hpp"
48
49
50 // These masks are used to provide 128-bit aligned bitmasks to the XMM
51 // instructions, to allow sign-masking or sign-bit flipping. They allow
52 // fast versions of NegF/NegD and AbsF/AbsD.
53
54 // Note: 'double' and 'long long' have 32-bits alignment on x86.
55 static jlong* double_quadword(jlong *adr, jlong lo, jlong hi) {
56 // Use the expression (adr)&(~0xF) to provide 128-bits aligned address
57 // of 128-bits operands for SSE instructions.
58 jlong *operand = (jlong*)(((intptr_t)adr) & ((intptr_t)(~0xF)));
59 // Store the value to a 128-bits operand.
60 operand[0] = lo;
428 __ bind(*stub->continuation());
429 }
430
431 if (compilation()->env()->dtrace_method_probes()) {
432 #ifdef _LP64
433 __ mov(rdi, r15_thread);
434 __ mov_metadata(rsi, method()->constant_encoding());
435 #else
436 __ get_thread(rax);
437 __ movptr(Address(rsp, 0), rax);
438 __ mov_metadata(Address(rsp, sizeof(void*)), method()->constant_encoding(), noreg);
439 #endif
440 __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit)));
441 }
442
443 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) {
444 __ mov(rax, rbx); // Restore the exception
445 }
446
447 // remove the activation and dispatch to the unwind handler
448 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
449 __ jump(RuntimeAddress(Runtime1::entry_for(C1StubId::unwind_exception_id)));
450
451 // Emit the slow path assembly
452 if (stub != nullptr) {
453 stub->emit_code(this);
454 }
455
456 return offset;
457 }
458
459
460 int LIR_Assembler::emit_deopt_handler() {
461 // generate code for exception handler
462 address handler_base = __ start_a_stub(deopt_handler_size());
463 if (handler_base == nullptr) {
464 // not enough space left for the handler
465 bailout("deopt handler overflow");
466 return -1;
467 }
468
469 int offset = code_offset();
470 InternalAddress here(__ pc());
471
472 __ pushptr(here.addr(), rscratch1);
473 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
474 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow");
475 __ end_a_stub();
476
477 return offset;
478 }
479
480 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) {
481 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == rax, "word returns are in rax,");
482 if (!result->is_illegal() && result->is_float_kind() && !result->is_xmm_register()) {
483 assert(result->fpu() == 0, "result must already be on TOS");
484 }
485 if (InlineTypeReturnedAsFields) {
486 #ifndef _LP64
487 Unimplemented();
488 #endif
489 // Check if we are returning an non-null inline type and load its fields into registers
490 ciType* return_type = compilation()->method()->return_type();
491 if (return_type->is_inlinetype()) {
492 ciInlineKlass* vk = return_type->as_inline_klass();
493 if (vk->can_be_returned_as_fields()) {
494 address unpack_handler = vk->unpack_handler();
495 assert(unpack_handler != nullptr, "must be");
496 __ call(RuntimeAddress(unpack_handler));
497 }
498 } else if (return_type->is_instance_klass() && (!return_type->is_loaded() || StressCallingConvention)) {
499 Label skip;
500 __ test_oop_is_not_inline_type(rax, rscratch1, skip);
501
502 // Load fields from a buffered value with an inline class specific handler
503 __ load_klass(rdi, rax, rscratch1);
504 __ movptr(rdi, Address(rdi, InstanceKlass::adr_inlineklass_fixed_block_offset()));
505 __ movptr(rdi, Address(rdi, InlineKlass::unpack_handler_offset()));
506 // Unpack handler can be null if inline type is not scalarizable in returns
507 __ testptr(rdi, rdi);
508 __ jcc(Assembler::zero, skip);
509 __ call(rdi);
510
511 __ bind(skip);
512 }
513 // At this point, rax points to the value object (for interpreter or C1 caller).
514 // The fields of the object are copied into registers (for C2 caller).
515 }
516
517 // Pop the stack before the safepoint code
518 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair());
519
520 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) {
521 __ reserved_stack_check();
522 }
523
524 // Note: we do not need to round double result; float result has the right precision
525 // the poll sets the condition code, but no data registers
526
527 #ifdef _LP64
528 const Register thread = r15_thread;
529 #else
530 const Register thread = rbx;
531 __ get_thread(thread);
532 #endif
533 code_stub->set_safepoint_offset(__ offset());
534 __ relocate(relocInfo::poll_return_type);
535 __ safepoint_poll(*code_stub->entry(), thread, true /* at_return */, true /* in_nmethod */);
536 __ ret(0);
537 }
538
539
540 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) {
541 return (__ store_inline_type_fields_to_buf(vk, false));
542 }
543
544 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) {
545 guarantee(info != nullptr, "Shouldn't be null");
546 int offset = __ offset();
547 #ifdef _LP64
548 const Register poll_addr = rscratch1;
549 __ movptr(poll_addr, Address(r15_thread, JavaThread::polling_page_offset()));
550 #else
551 assert(tmp->is_cpu_register(), "needed");
552 const Register poll_addr = tmp->as_register();
553 __ get_thread(poll_addr);
554 __ movptr(poll_addr, Address(poll_addr, in_bytes(JavaThread::polling_page_offset())));
555 #endif
556 add_debug_info_for_branch(info);
557 __ relocate(relocInfo::poll_type);
558 address pre_pc = __ pc();
559 __ testl(rax, Address(poll_addr, 0));
560 address post_pc = __ pc();
561 guarantee(pointer_delta(post_pc, pre_pc, 1) == 2 LP64_ONLY(+1), "must be exact length");
562 return offset;
563 }
1418 // init_state needs acquire, but x86 is TSO, and so we are already good.
1419 __ cmpb(Address(op->klass()->as_register(),
1420 InstanceKlass::init_state_offset()),
1421 InstanceKlass::fully_initialized);
1422 __ jcc(Assembler::notEqual, *op->stub()->entry());
1423 }
1424 __ allocate_object(op->obj()->as_register(),
1425 op->tmp1()->as_register(),
1426 op->tmp2()->as_register(),
1427 op->header_size(),
1428 op->object_size(),
1429 op->klass()->as_register(),
1430 *op->stub()->entry());
1431 __ bind(*op->stub()->continuation());
1432 }
1433
1434 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) {
1435 Register len = op->len()->as_register();
1436 LP64_ONLY( __ movslq(len, len); )
1437
1438 if (UseSlowPath || op->is_null_free() ||
1439 (!UseFastNewObjectArray && is_reference_type(op->type())) ||
1440 (!UseFastNewTypeArray && !is_reference_type(op->type()))) {
1441 __ jmp(*op->stub()->entry());
1442 } else {
1443 Register tmp1 = op->tmp1()->as_register();
1444 Register tmp2 = op->tmp2()->as_register();
1445 Register tmp3 = op->tmp3()->as_register();
1446 if (len == tmp1) {
1447 tmp1 = tmp3;
1448 } else if (len == tmp2) {
1449 tmp2 = tmp3;
1450 } else if (len == tmp3) {
1451 // everything is ok
1452 } else {
1453 __ mov(tmp3, len);
1454 }
1455 __ allocate_array(op->obj()->as_register(),
1456 len,
1457 tmp1,
1458 tmp2,
1517 assert(data != nullptr, "need data for type check");
1518 assert(data->is_ReceiverTypeData(), "need ReceiverTypeData for type check");
1519 }
1520 Label* success_target = success;
1521 Label* failure_target = failure;
1522
1523 if (obj == k_RInfo) {
1524 k_RInfo = dst;
1525 } else if (obj == klass_RInfo) {
1526 klass_RInfo = dst;
1527 }
1528 if (k->is_loaded() && !UseCompressedClassPointers) {
1529 select_different_registers(obj, dst, k_RInfo, klass_RInfo);
1530 } else {
1531 Rtmp1 = op->tmp3()->as_register();
1532 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1);
1533 }
1534
1535 assert_different_registers(obj, k_RInfo, klass_RInfo);
1536
1537 if (op->need_null_check()) {
1538 __ testptr(obj, obj);
1539 if (op->should_profile()) {
1540 Label not_null;
1541 Register mdo = klass_RInfo;
1542 __ mov_metadata(mdo, md->constant_encoding());
1543 __ jccb(Assembler::notEqual, not_null);
1544 // Object is null; update MDO and exit
1545 Address data_addr(mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset()));
1546 int header_bits = BitData::null_seen_byte_constant();
1547 __ orb(data_addr, header_bits);
1548 __ jmp(*obj_is_null);
1549 __ bind(not_null);
1550
1551 Label update_done;
1552 Register recv = k_RInfo;
1553 __ load_klass(recv, obj, tmp_load_klass);
1554 type_profile_helper(mdo, md, data, recv, &update_done);
1555
1556 Address nonprofiled_receiver_count_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset()));
1557 __ addptr(nonprofiled_receiver_count_addr, DataLayout::counter_increment);
1558
1559 __ bind(update_done);
1560 } else {
1561 __ jcc(Assembler::equal, *obj_is_null);
1562 }
1563 }
1564
1565 if (!k->is_loaded()) {
1566 klass2reg_with_patching(k_RInfo, op->info_for_patch());
1567 } else {
1568 #ifdef _LP64
1569 __ mov_metadata(k_RInfo, k->constant_encoding());
1570 #endif // _LP64
1571 }
1572 __ verify_oop(obj);
1573
1574 if (op->fast_check()) {
1575 // get object class
1576 // not a safepoint as obj null check happens earlier
1577 #ifdef _LP64
1578 if (UseCompressedClassPointers) {
1579 __ load_klass(Rtmp1, obj, tmp_load_klass);
1580 __ cmpptr(k_RInfo, Rtmp1);
1581 } else {
1582 __ cmpptr(k_RInfo, Address(obj, oopDesc::klass_offset_in_bytes()));
1734 __ mov(dst, obj);
1735 }
1736 } else
1737 if (code == lir_instanceof) {
1738 Register obj = op->object()->as_register();
1739 Register dst = op->result_opr()->as_register();
1740 Label success, failure, done;
1741 emit_typecheck_helper(op, &success, &failure, &failure);
1742 __ bind(failure);
1743 __ xorptr(dst, dst);
1744 __ jmpb(done);
1745 __ bind(success);
1746 __ movptr(dst, 1);
1747 __ bind(done);
1748 } else {
1749 ShouldNotReachHere();
1750 }
1751
1752 }
1753
1754 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) {
1755 // We are loading/storing from/to an array that *may* be a flat array (the
1756 // declared type is Object[], abstract[], interface[] or VT.ref[]).
1757 // If this array is a flat array, take the slow path.
1758 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1759 if (!op->value()->is_illegal()) {
1760 // TODO 8350865 This is also used for profiling code, right? And in that case we don't care about null but just want to know if the array is flat or not.
1761 // The array is not a flat array, but it might be null-free. If we are storing
1762 // a null into a null-free array, take the slow path (which will throw NPE).
1763 Label skip;
1764 __ cmpptr(op->value()->as_register(), NULL_WORD);
1765 __ jcc(Assembler::notEqual, skip);
1766 __ test_null_free_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry());
1767 __ bind(skip);
1768 }
1769 }
1770
1771 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) {
1772 // We are storing into an array that *may* be null-free (the declared type is
1773 // Object[], abstract[], interface[] or VT.ref[]).
1774 Label test_mark_word;
1775 Register tmp = op->tmp()->as_register();
1776 __ movptr(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes()));
1777 __ testl(tmp, markWord::unlocked_value);
1778 __ jccb(Assembler::notZero, test_mark_word);
1779 __ load_prototype_header(tmp, op->array()->as_register(), rscratch1);
1780 __ bind(test_mark_word);
1781 __ testl(tmp, markWord::null_free_array_bit_in_place);
1782 }
1783
1784 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) {
1785 Label L_oops_equal;
1786 Label L_oops_not_equal;
1787 Label L_end;
1788
1789 Register left = op->left()->as_register();
1790 Register right = op->right()->as_register();
1791
1792 __ cmpptr(left, right);
1793 __ jcc(Assembler::equal, L_oops_equal);
1794
1795 // (1) Null check -- if one of the operands is null, the other must not be null (because
1796 // the two references are not equal), so they are not substitutable,
1797 // FIXME: do null check only if the operand is nullable
1798 __ testptr(left, right);
1799 __ jcc(Assembler::zero, L_oops_not_equal);
1800
1801 ciKlass* left_klass = op->left_klass();
1802 ciKlass* right_klass = op->right_klass();
1803
1804 // (2) Inline type check -- if either of the operands is not a inline type,
1805 // they are not substitutable. We do this only if we are not sure that the
1806 // operands are inline type
1807 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node.
1808 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) {
1809 Register tmp1 = op->tmp1()->as_register();
1810 __ movptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1811 __ andptr(tmp1, Address(left, oopDesc::mark_offset_in_bytes()));
1812 __ andptr(tmp1, Address(right, oopDesc::mark_offset_in_bytes()));
1813 __ cmpptr(tmp1, (intptr_t)markWord::inline_type_pattern);
1814 __ jcc(Assembler::notEqual, L_oops_not_equal);
1815 }
1816
1817 // (3) Same klass check: if the operands are of different klasses, they are not substitutable.
1818 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) {
1819 // No need to load klass -- the operands are statically known to be the same inline klass.
1820 __ jmp(*op->stub()->entry());
1821 } else {
1822 Register left_klass_op = op->left_klass_op()->as_register();
1823 Register right_klass_op = op->right_klass_op()->as_register();
1824
1825 if (UseCompressedClassPointers) {
1826 __ movl(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1827 __ movl(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1828 __ cmpl(left_klass_op, right_klass_op);
1829 } else {
1830 __ movptr(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes()));
1831 __ movptr(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes()));
1832 __ cmpptr(left_klass_op, right_klass_op);
1833 }
1834
1835 __ jcc(Assembler::equal, *op->stub()->entry()); // same klass -> do slow check
1836 // fall through to L_oops_not_equal
1837 }
1838
1839 __ bind(L_oops_not_equal);
1840 move(op->not_equal_result(), op->result_opr());
1841 __ jmp(L_end);
1842
1843 __ bind(L_oops_equal);
1844 move(op->equal_result(), op->result_opr());
1845 __ jmp(L_end);
1846
1847 // We've returned from the stub. RAX contains 0x0 IFF the two
1848 // operands are not substitutable. (Don't compare against 0x1 in case the
1849 // C compiler is naughty)
1850 __ bind(*op->stub()->continuation());
1851 __ cmpl(rax, 0);
1852 __ jcc(Assembler::equal, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal
1853 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal
1854 // fall-through
1855 __ bind(L_end);
1856 }
1857
1858 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) {
1859 if (LP64_ONLY(false &&) op->code() == lir_cas_long) {
1860 assert(op->cmp_value()->as_register_lo() == rax, "wrong register");
1861 assert(op->cmp_value()->as_register_hi() == rdx, "wrong register");
1862 assert(op->new_value()->as_register_lo() == rbx, "wrong register");
1863 assert(op->new_value()->as_register_hi() == rcx, "wrong register");
1864 Register addr = op->addr()->as_register();
1865 __ lock();
1866 NOT_LP64(__ cmpxchg8(Address(addr, 0)));
1867
1868 } else if (op->code() == lir_cas_int || op->code() == lir_cas_obj ) {
1869 NOT_LP64(assert(op->addr()->is_single_cpu(), "must be single");)
1870 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1871 Register newval = op->new_value()->as_register();
1872 Register cmpval = op->cmp_value()->as_register();
1873 assert(cmpval == rax, "wrong register");
1874 assert(newval != noreg, "new val must be register");
1875 assert(cmpval != newval, "cmp and new values must be in different registers");
1876 assert(cmpval != addr, "cmp and addr must be in different registers");
1897 __ cmpxchgl(newval, Address(addr, 0));
1898 }
1899 #ifdef _LP64
1900 } else if (op->code() == lir_cas_long) {
1901 Register addr = (op->addr()->is_single_cpu() ? op->addr()->as_register() : op->addr()->as_register_lo());
1902 Register newval = op->new_value()->as_register_lo();
1903 Register cmpval = op->cmp_value()->as_register_lo();
1904 assert(cmpval == rax, "wrong register");
1905 assert(newval != noreg, "new val must be register");
1906 assert(cmpval != newval, "cmp and new values must be in different registers");
1907 assert(cmpval != addr, "cmp and addr must be in different registers");
1908 assert(newval != addr, "new value and addr must be in different registers");
1909 __ lock();
1910 __ cmpxchgq(newval, Address(addr, 0));
1911 #endif // _LP64
1912 } else {
1913 Unimplemented();
1914 }
1915 }
1916
1917 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) {
1918 assert(dst->is_cpu_register(), "must be");
1919 assert(dst->type() == src->type(), "must be");
1920
1921 if (src->is_cpu_register()) {
1922 reg2reg(src, dst);
1923 } else if (src->is_stack()) {
1924 stack2reg(src, dst, dst->type());
1925 } else if (src->is_constant()) {
1926 const2reg(src, dst, lir_patch_none, nullptr);
1927 } else {
1928 ShouldNotReachHere();
1929 }
1930 }
1931
1932 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type,
1933 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) {
1934 assert(cmp_opr1 == LIR_OprFact::illegalOpr && cmp_opr2 == LIR_OprFact::illegalOpr, "unnecessary cmp oprs on x86");
1935
1936 Assembler::Condition acond, ncond;
1937 switch (condition) {
1938 case lir_cond_equal: acond = Assembler::equal; ncond = Assembler::notEqual; break;
1939 case lir_cond_notEqual: acond = Assembler::notEqual; ncond = Assembler::equal; break;
1940 case lir_cond_less: acond = Assembler::less; ncond = Assembler::greaterEqual; break;
1941 case lir_cond_lessEqual: acond = Assembler::lessEqual; ncond = Assembler::greater; break;
1942 case lir_cond_greaterEqual: acond = Assembler::greaterEqual; ncond = Assembler::less; break;
1943 case lir_cond_greater: acond = Assembler::greater; ncond = Assembler::lessEqual; break;
1944 case lir_cond_belowEqual: acond = Assembler::belowEqual; ncond = Assembler::above; break;
1945 case lir_cond_aboveEqual: acond = Assembler::aboveEqual; ncond = Assembler::below; break;
1946 default: acond = Assembler::equal; ncond = Assembler::notEqual;
1947 ShouldNotReachHere();
1948 }
1949
1950 if (opr1->is_cpu_register()) {
1951 reg2reg(opr1, result);
2624 int offset = __ offset();
2625 switch (code) {
2626 case lir_static_call:
2627 case lir_optvirtual_call:
2628 case lir_dynamic_call:
2629 offset += NativeCall::displacement_offset;
2630 break;
2631 case lir_icvirtual_call:
2632 offset += NativeCall::displacement_offset + NativeMovConstReg::instruction_size_rex;
2633 break;
2634 default: ShouldNotReachHere();
2635 }
2636 __ align(BytesPerWord, offset);
2637 }
2638
2639
2640 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) {
2641 assert((__ offset() + NativeCall::displacement_offset) % BytesPerWord == 0,
2642 "must be aligned");
2643 __ call(AddressLiteral(op->addr(), rtype));
2644 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2645 __ post_call_nop();
2646 }
2647
2648
2649 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) {
2650 __ ic_call(op->addr());
2651 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields());
2652 assert((__ offset() - NativeCall::instruction_size + NativeCall::displacement_offset) % BytesPerWord == 0,
2653 "must be aligned");
2654 __ post_call_nop();
2655 }
2656
2657
2658 void LIR_Assembler::emit_static_call_stub() {
2659 address call_pc = __ pc();
2660 address stub = __ start_a_stub(call_stub_size());
2661 if (stub == nullptr) {
2662 bailout("static call stub overflow");
2663 return;
2664 }
2665
2666 int start = __ offset();
2667
2668 // make sure that the displacement word of the call ends up word aligned
2669 __ align(BytesPerWord, __ offset() + NativeMovConstReg::instruction_size_rex + NativeCall::displacement_offset);
2670 __ relocate(static_stub_Relocation::spec(call_pc));
2671 __ mov_metadata(rbx, (Metadata*)nullptr);
2812 __ movptr (Address(rsp, offset_from_rsp_in_bytes), c);
2813 }
2814
2815
2816 void LIR_Assembler::store_parameter(jobject o, int offset_from_rsp_in_words) {
2817 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2818 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2819 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2820 __ movoop(Address(rsp, offset_from_rsp_in_bytes), o, rscratch1);
2821 }
2822
2823
2824 void LIR_Assembler::store_parameter(Metadata* m, int offset_from_rsp_in_words) {
2825 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp");
2826 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord;
2827 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset");
2828 __ mov_metadata(Address(rsp, offset_from_rsp_in_bytes), m, rscratch1);
2829 }
2830
2831
2832 void LIR_Assembler::arraycopy_inlinetype_check(Register obj, Register tmp, CodeStub* slow_path, bool is_dest, bool null_check) {
2833 if (null_check) {
2834 __ testptr(obj, obj);
2835 __ jcc(Assembler::zero, *slow_path->entry());
2836 }
2837 if (is_dest) {
2838 __ test_null_free_array_oop(obj, tmp, *slow_path->entry());
2839 // TODO 8350865 Flat no longer implies null-free, so we need to check for flat dest. Can we do better here?
2840 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2841 } else {
2842 __ test_flat_array_oop(obj, tmp, *slow_path->entry());
2843 }
2844 }
2845
2846
2847 // This code replaces a call to arraycopy; no exception may
2848 // be thrown in this code, they must be thrown in the System.arraycopy
2849 // activation frame; we could save some checks if this would not be the case
2850 void LIR_Assembler::emit_arraycopy(LIR_OpArrayCopy* op) {
2851 ciArrayKlass* default_type = op->expected_type();
2852 Register src = op->src()->as_register();
2853 Register dst = op->dst()->as_register();
2854 Register src_pos = op->src_pos()->as_register();
2855 Register dst_pos = op->dst_pos()->as_register();
2856 Register length = op->length()->as_register();
2857 Register tmp = op->tmp()->as_register();
2858 Register tmp_load_klass = LP64_ONLY(rscratch1) NOT_LP64(noreg);
2859 Register tmp2 = UseCompactObjectHeaders ? rscratch2 : noreg;
2860
2861 CodeStub* stub = op->stub();
2862 int flags = op->flags();
2863 BasicType basic_type = default_type != nullptr ? default_type->element_type()->basic_type() : T_ILLEGAL;
2864 if (is_reference_type(basic_type)) basic_type = T_OBJECT;
2865
2866 if (flags & LIR_OpArrayCopy::always_slow_path) {
2867 __ jmp(*stub->entry());
2868 __ bind(*stub->continuation());
2869 return;
2870 }
2871
2872 // if we don't know anything, just go through the generic arraycopy
2873 if (default_type == nullptr) {
2874 // save outgoing arguments on stack in case call to System.arraycopy is needed
2875 // HACK ALERT. This code used to push the parameters in a hardwired fashion
2876 // for interpreter calling conventions. Now we have to do it in new style conventions.
2877 // For the moment until C1 gets the new register allocator I just force all the
2878 // args to the right place (except the register args) and then on the back side
2879 // reload the register args properly if we go slow path. Yuck
2880
2881 // These are proper for the calling convention
2882 store_parameter(length, 2);
2883 store_parameter(dst_pos, 1);
2884 store_parameter(dst, 0);
2885
2886 // these are just temporary placements until we need to reload
2887 store_parameter(src_pos, 3);
2888 store_parameter(src, 4);
2889 NOT_LP64(assert(src == rcx && src_pos == rdx, "mismatch in calling convention");)
2890
2891 address copyfunc_addr = StubRoutines::generic_arraycopy();
2945 __ mov(tmp, rax);
2946 __ xorl(tmp, -1);
2947
2948 // Reload values from the stack so they are where the stub
2949 // expects them.
2950 __ movptr (dst, Address(rsp, 0*BytesPerWord));
2951 __ movptr (dst_pos, Address(rsp, 1*BytesPerWord));
2952 __ movptr (length, Address(rsp, 2*BytesPerWord));
2953 __ movptr (src_pos, Address(rsp, 3*BytesPerWord));
2954 __ movptr (src, Address(rsp, 4*BytesPerWord));
2955
2956 __ subl(length, tmp);
2957 __ addl(src_pos, tmp);
2958 __ addl(dst_pos, tmp);
2959 __ jmp(*stub->entry());
2960
2961 __ bind(*stub->continuation());
2962 return;
2963 }
2964
2965 // Handle inline type arrays
2966 if (flags & LIR_OpArrayCopy::src_inlinetype_check) {
2967 arraycopy_inlinetype_check(src, tmp, stub, false, (flags & LIR_OpArrayCopy::src_null_check));
2968 }
2969 if (flags & LIR_OpArrayCopy::dst_inlinetype_check) {
2970 arraycopy_inlinetype_check(dst, tmp, stub, true, (flags & LIR_OpArrayCopy::dst_null_check));
2971 }
2972
2973 assert(default_type != nullptr && default_type->is_array_klass() && default_type->is_loaded(), "must be true at this point");
2974
2975 int elem_size = type2aelembytes(basic_type);
2976 Address::ScaleFactor scale;
2977
2978 switch (elem_size) {
2979 case 1 :
2980 scale = Address::times_1;
2981 break;
2982 case 2 :
2983 scale = Address::times_2;
2984 break;
2985 case 4 :
2986 scale = Address::times_4;
2987 break;
2988 case 8 :
2989 scale = Address::times_8;
2990 break;
2991 default:
2992 scale = Address::no_scale;
3565 // first time here. Set profile type.
3566 __ movptr(mdo_addr, tmp);
3567 #ifdef ASSERT
3568 __ andptr(tmp, TypeEntries::type_klass_mask);
3569 __ verify_klass_ptr(tmp);
3570 #endif
3571 } else {
3572 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr &&
3573 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent");
3574
3575 __ testptr(mdo_addr, TypeEntries::type_unknown);
3576 __ jccb(Assembler::notZero, next); // already unknown. Nothing to do anymore.
3577
3578 __ orptr(mdo_addr, TypeEntries::type_unknown);
3579 }
3580 }
3581 }
3582 __ bind(next);
3583 }
3584
3585 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) {
3586 Register obj = op->obj()->as_register();
3587 Register tmp = op->tmp()->as_pointer_register();
3588 Address mdo_addr = as_Address(op->mdp()->as_address_ptr());
3589 bool not_null = op->not_null();
3590 int flag = op->flag();
3591
3592 Label not_inline_type;
3593 if (!not_null) {
3594 __ testptr(obj, obj);
3595 __ jccb(Assembler::zero, not_inline_type);
3596 }
3597
3598 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type);
3599
3600 __ orb(mdo_addr, flag);
3601
3602 __ bind(not_inline_type);
3603 }
3604
3605 void LIR_Assembler::emit_delay(LIR_OpDelay*) {
3606 Unimplemented();
3607 }
3608
3609
3610 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) {
3611 __ lea(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no));
3612 }
3613
3614
3615 void LIR_Assembler::align_backward_branch_target() {
3616 __ align(BytesPerWord);
3617 }
3618
3619
3620 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) {
3621 if (left->is_single_cpu()) {
3622 __ negl(left->as_register());
3623 move_regs(left->as_register(), dest->as_register());
3624
3798 }
3799
3800 void LIR_Assembler::membar_storeload() {
3801 __ membar(Assembler::Membar_mask_bits(Assembler::StoreLoad));
3802 }
3803
3804 void LIR_Assembler::on_spin_wait() {
3805 __ pause ();
3806 }
3807
3808 void LIR_Assembler::get_thread(LIR_Opr result_reg) {
3809 assert(result_reg->is_register(), "check");
3810 #ifdef _LP64
3811 // __ get_thread(result_reg->as_register_lo());
3812 __ mov(result_reg->as_register(), r15_thread);
3813 #else
3814 __ get_thread(result_reg->as_register());
3815 #endif // _LP64
3816 }
3817
3818 void LIR_Assembler::check_orig_pc() {
3819 __ cmpptr(frame_map()->address_for_orig_pc_addr(), NULL_WORD);
3820 }
3821
3822 void LIR_Assembler::peephole(LIR_List*) {
3823 // do nothing for now
3824 }
3825
3826 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp) {
3827 assert(data == dest, "xchg/xadd uses only 2 operands");
3828
3829 if (data->type() == T_INT) {
3830 if (code == lir_xadd) {
3831 __ lock();
3832 __ xaddl(as_Address(src->as_address_ptr()), data->as_register());
3833 } else {
3834 __ xchgl(data->as_register(), as_Address(src->as_address_ptr()));
3835 }
3836 } else if (data->is_oop()) {
3837 assert (code == lir_xchg, "xadd for oops");
3838 Register obj = data->as_register();
3839 #ifdef _LP64
3840 if (UseCompressedOops) {
|