1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
2207 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2208 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2209 int reg = ra_->get_encode(this);
2210
2211 // This add will handle any 24-bit signed offset. 24 bits allows an
2212 // 8 megabyte stack frame.
2213 __ add(as_Register(reg), sp, offset);
2214 }
2215
2216 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2217 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2218 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2219
2220 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2221 return NativeInstruction::instruction_size;
2222 } else {
2223 return 2 * NativeInstruction::instruction_size;
2224 }
2225 }
2226
2227 //=============================================================================
2228
2229 #ifndef PRODUCT
2230 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2231 {
2232 st->print_cr("# MachUEPNode");
2233 if (UseCompressedClassPointers) {
2234 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2235 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2236 st->print_cr("\tcmpw rscratch1, r10");
2237 } else {
2238 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2239 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2240 st->print_cr("\tcmp rscratch1, r10");
2241 }
2242 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2243 }
2244 #endif
2245
2246 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2247 {
2248 __ ic_check(InteriorEntryAlignment);
2249 }
2250
2251 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2252 {
2253 return MachNode::size(ra_);
2254 }
2255
2256 // REQUIRED EMIT CODE
2257
2258 //=============================================================================
2259
2260 // Emit deopt handler code.
2261 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2262 {
2263 // Note that the code buffer's insts_mark is always relative to insts.
2264 // That's why we must use the macroassembler to generate a handler.
2265 address base = __ start_a_stub(size_deopt_handler());
2266 if (base == nullptr) {
2267 ciEnv::current()->record_failure("CodeCache is full");
2268 return 0; // CodeBuffer::expand failed
2269 }
2270
2271 int offset = __ offset();
2272 Label start;
2273 __ bind(start);
2274 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2275
3722 %}
3723
3724 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3725 int method_index = resolved_method_index(masm);
3726 address call = __ ic_call((address)$meth$$method, method_index);
3727 if (call == nullptr) {
3728 ciEnv::current()->record_failure("CodeCache is full");
3729 return;
3730 }
3731 __ post_call_nop();
3732 if (Compile::current()->max_vector_size() > 0) {
3733 __ reinitialize_ptrue();
3734 }
3735 %}
3736
3737 enc_class aarch64_enc_call_epilog() %{
3738 if (VerifyStackAtCalls) {
3739 // Check that stack depth is unchanged: find majik cookie on stack
3740 __ call_Unimplemented();
3741 }
3742 %}
3743
3744 enc_class aarch64_enc_java_to_runtime(method meth) %{
3745 // some calls to generated routines (arraycopy code) are scheduled
3746 // by C2 as runtime calls. if so we can call them using a br (they
3747 // will be in a reachable segment) otherwise we have to use a blr
3748 // which loads the absolute address into a register.
3749 address entry = (address)$meth$$method;
3750 CodeBlob *cb = CodeCache::find_blob(entry);
3751 if (cb) {
3752 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3753 if (call == nullptr) {
3754 ciEnv::current()->record_failure("CodeCache is full");
3755 return;
3756 }
3757 __ post_call_nop();
3758 } else {
3759 Label retaddr;
3760 // Make the anchor frame walkable
3761 __ adr(rscratch2, retaddr);
6903 instruct loadConL(iRegLNoSp dst, immL src)
6904 %{
6905 match(Set dst src);
6906
6907 ins_cost(INSN_COST);
6908 format %{ "mov $dst, $src\t# long" %}
6909
6910 ins_encode( aarch64_enc_mov_imm(dst, src) );
6911
6912 ins_pipe(ialu_imm);
6913 %}
6914
6915 // Load Pointer Constant
6916
6917 instruct loadConP(iRegPNoSp dst, immP con)
6918 %{
6919 match(Set dst con);
6920
6921 ins_cost(INSN_COST * 4);
6922 format %{
6923 "mov $dst, $con\t# ptr\n\t"
6924 %}
6925
6926 ins_encode(aarch64_enc_mov_p(dst, con));
6927
6928 ins_pipe(ialu_imm);
6929 %}
6930
6931 // Load Null Pointer Constant
6932
6933 instruct loadConP0(iRegPNoSp dst, immP0 con)
6934 %{
6935 match(Set dst con);
6936
6937 ins_cost(INSN_COST);
6938 format %{ "mov $dst, $con\t# nullptr ptr" %}
6939
6940 ins_encode(aarch64_enc_mov_p0(dst, con));
6941
6942 ins_pipe(ialu_imm);
6943 %}
8096 %}
8097
8098 // ============================================================================
8099 // Cast/Convert Instructions
8100
8101 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8102 match(Set dst (CastX2P src));
8103
8104 ins_cost(INSN_COST);
8105 format %{ "mov $dst, $src\t# long -> ptr" %}
8106
8107 ins_encode %{
8108 if ($dst$$reg != $src$$reg) {
8109 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8110 }
8111 %}
8112
8113 ins_pipe(ialu_reg);
8114 %}
8115
8116 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8117 match(Set dst (CastP2X src));
8118
8119 ins_cost(INSN_COST);
8120 format %{ "mov $dst, $src\t# ptr -> long" %}
8121
8122 ins_encode %{
8123 if ($dst$$reg != $src$$reg) {
8124 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8125 }
8126 %}
8127
8128 ins_pipe(ialu_reg);
8129 %}
8130
8131 // Convert oop into int for vectors alignment masking
8132 instruct convP2I(iRegINoSp dst, iRegP src) %{
8133 match(Set dst (ConvL2I (CastP2X src)));
8134
8135 ins_cost(INSN_COST);
15049
15050 match(Set dst (MoveL2D src));
15051
15052 effect(DEF dst, USE src);
15053
15054 ins_cost(INSN_COST);
15055
15056 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15057
15058 ins_encode %{
15059 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15060 %}
15061
15062 ins_pipe(fp_l2d);
15063
15064 %}
15065
15066 // ============================================================================
15067 // clearing of an array
15068
15069 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
15070 %{
15071 match(Set dummy (ClearArray cnt base));
15072 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15073
15074 ins_cost(4 * INSN_COST);
15075 format %{ "ClearArray $cnt, $base" %}
15076
15077 ins_encode %{
15078 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15079 if (tpc == nullptr) {
15080 ciEnv::current()->record_failure("CodeCache is full");
15081 return;
15082 }
15083 %}
15084
15085 ins_pipe(pipe_class_memory);
15086 %}
15087
15088 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15089 %{
15090 predicate((uint64_t)n->in(2)->get_long()
15091 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
15092 match(Set dummy (ClearArray cnt base));
15093 effect(TEMP temp, USE_KILL base, KILL cr);
15094
15095 ins_cost(4 * INSN_COST);
15096 format %{ "ClearArray $cnt, $base" %}
15097
15098 ins_encode %{
15099 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15100 if (tpc == nullptr) {
15101 ciEnv::current()->record_failure("CodeCache is full");
15102 return;
15103 }
15104 %}
15105
15106 ins_pipe(pipe_class_memory);
15107 %}
15108
15109 // ============================================================================
15110 // Overflow Math Instructions
15111
16388 %}
16389
16390 // Call Runtime Instruction without safepoint and with vector arguments
16391 instruct CallLeafDirectVector(method meth)
16392 %{
16393 match(CallLeafVector);
16394
16395 effect(USE meth);
16396
16397 ins_cost(CALL_COST);
16398
16399 format %{ "CALL, runtime leaf vector $meth" %}
16400
16401 ins_encode(aarch64_enc_java_to_runtime(meth));
16402
16403 ins_pipe(pipe_class_call);
16404 %}
16405
16406 // Call Runtime Instruction
16407
16408 instruct CallLeafNoFPDirect(method meth)
16409 %{
16410 match(CallLeafNoFP);
16411
16412 effect(USE meth);
16413
16414 ins_cost(CALL_COST);
16415
16416 format %{ "CALL, runtime leaf nofp $meth" %}
16417
16418 ins_encode( aarch64_enc_java_to_runtime(meth) );
16419
16420 ins_pipe(pipe_class_call);
16421 %}
16422
16423 // Tail Call; Jump from runtime stub to Java code.
16424 // Also known as an 'interprocedural jump'.
16425 // Target of jump will eventually return to caller.
16426 // TailJump below removes the return address.
16427 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16428 // emitted just above the TailCall which has reset rfp to the caller state.
16429 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
|
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else if (_entry_point == nullptr) {
1694 // See CallLeafNoFPIndirect
1695 return 1 * NativeInstruction::instruction_size;
1696 } else {
1697 return 6 * NativeInstruction::instruction_size;
1698 }
1699 }
1700
1701 //=============================================================================
1702
1703 #ifndef PRODUCT
1704 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1705 st->print("BREAKPOINT");
1706 }
1707 #endif
1708
1709 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1710 __ brk(0);
1711 }
1712
1713 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1714 return MachNode::size(ra_);
1715 }
1784 if (C->stub_function() == nullptr) {
1785 st->print("\n\t");
1786 st->print("ldr rscratch1, [guard]\n\t");
1787 st->print("dmb ishld\n\t");
1788 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1789 st->print("cmp rscratch1, rscratch2\n\t");
1790 st->print("b.eq skip");
1791 st->print("\n\t");
1792 st->print("blr #nmethod_entry_barrier_stub\n\t");
1793 st->print("b skip\n\t");
1794 st->print("guard: int\n\t");
1795 st->print("\n\t");
1796 st->print("skip:\n\t");
1797 }
1798 }
1799 #endif
1800
1801 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1802 Compile* C = ra_->C;
1803
1804
1805 __ verified_entry(C, 0);
1806
1807 if (C->stub_function() == nullptr) {
1808 __ entry_barrier();
1809 }
1810
1811 if (!Compile::current()->output()->in_scratch_emit_size()) {
1812 __ bind(*_verified_entry);
1813 }
1814
1815 if (VerifyStackAtCalls) {
1816 Unimplemented();
1817 }
1818
1819 C->output()->set_frame_complete(__ offset());
1820
1821 if (C->has_mach_constant_base_node()) {
1822 // NOTE: We set the table base offset here because users might be
1823 // emitted before MachConstantBaseNode.
1824 ConstantTable& constant_table = C->output()->constant_table();
1825 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1826 }
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autiaz\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1870
1871 __ remove_frame(framesize, C->needs_stack_repair());
1872
1873 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1874 __ reserved_stack_check();
1875 }
1876
1877 if (do_polling() && C->is_method_compilation()) {
1878 Label dummy_label;
1879 Label* code_stub = &dummy_label;
1880 if (!C->output()->in_scratch_emit_size()) {
1881 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1882 C->output()->add_stub(stub);
1883 code_stub = &stub->entry();
1884 }
1885 __ relocate(relocInfo::poll_return_type);
1886 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1887 }
1888 }
1889
1890 int MachEpilogNode::reloc() const {
1891 // Return number of relocatable values contained in this instruction.
1892 return 1; // 1 for polling page.
1893 }
1894
1895 const Pipeline * MachEpilogNode::pipeline() const {
1896 return MachNode::pipeline_class();
1897 }
1898
1899 //=============================================================================
1900
1901 static enum RC rc_class(OptoReg::Name reg) {
1902
1903 if (reg == OptoReg::Bad) {
1904 return rc_bad;
1905 }
1906
1907 // we have 32 int registers * 2 halves
1908 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1909
2165 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2166 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2167 int reg = ra_->get_encode(this);
2168
2169 // This add will handle any 24-bit signed offset. 24 bits allows an
2170 // 8 megabyte stack frame.
2171 __ add(as_Register(reg), sp, offset);
2172 }
2173
2174 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2175 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2176 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2177
2178 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2179 return NativeInstruction::instruction_size;
2180 } else {
2181 return 2 * NativeInstruction::instruction_size;
2182 }
2183 }
2184
2185 ///=============================================================================
2186 #ifndef PRODUCT
2187 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2188 {
2189 st->print_cr("# MachVEPNode");
2190 if (!_verified) {
2191 st->print_cr("\t load_class");
2192 } else {
2193 st->print_cr("\t unpack_inline_arg");
2194 }
2195 }
2196 #endif
2197
2198 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2199 {
2200 if (!_verified) {
2201 __ ic_check(1);
2202 } else {
2203 // TODO 8284443 Avoid creation of temporary frame
2204 if (ra_->C->stub_function() == nullptr) {
2205 __ verified_entry(ra_->C, 0);
2206 __ entry_barrier();
2207 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2208 __ remove_frame(framesize, false);
2209 }
2210 // Unpack inline type args passed as oop and then jump to
2211 // the verified entry point (skipping the unverified entry).
2212 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2213 // Emit code for verified entry and save increment for stack repair on return
2214 __ verified_entry(ra_->C, sp_inc);
2215 if (Compile::current()->output()->in_scratch_emit_size()) {
2216 Label dummy_verified_entry;
2217 __ b(dummy_verified_entry);
2218 } else {
2219 __ b(*_verified_entry);
2220 }
2221 }
2222 }
2223
2224 //=============================================================================
2225 #ifndef PRODUCT
2226 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2227 {
2228 st->print_cr("# MachUEPNode");
2229 if (UseCompressedClassPointers) {
2230 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2231 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2232 st->print_cr("\tcmpw rscratch1, r10");
2233 } else {
2234 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2235 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2236 st->print_cr("\tcmp rscratch1, r10");
2237 }
2238 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2239 }
2240 #endif
2241
2242 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2243 {
2244 __ ic_check(InteriorEntryAlignment);
2245 }
2246
2247 // REQUIRED EMIT CODE
2248
2249 //=============================================================================
2250
2251 // Emit deopt handler code.
2252 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2253 {
2254 // Note that the code buffer's insts_mark is always relative to insts.
2255 // That's why we must use the macroassembler to generate a handler.
2256 address base = __ start_a_stub(size_deopt_handler());
2257 if (base == nullptr) {
2258 ciEnv::current()->record_failure("CodeCache is full");
2259 return 0; // CodeBuffer::expand failed
2260 }
2261
2262 int offset = __ offset();
2263 Label start;
2264 __ bind(start);
2265 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2266
3713 %}
3714
3715 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3716 int method_index = resolved_method_index(masm);
3717 address call = __ ic_call((address)$meth$$method, method_index);
3718 if (call == nullptr) {
3719 ciEnv::current()->record_failure("CodeCache is full");
3720 return;
3721 }
3722 __ post_call_nop();
3723 if (Compile::current()->max_vector_size() > 0) {
3724 __ reinitialize_ptrue();
3725 }
3726 %}
3727
3728 enc_class aarch64_enc_call_epilog() %{
3729 if (VerifyStackAtCalls) {
3730 // Check that stack depth is unchanged: find majik cookie on stack
3731 __ call_Unimplemented();
3732 }
3733 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3734 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3735 // Search for the corresponding projection, get the register and emit code that initialized it.
3736 uint con = (tf()->range_cc()->cnt() - 1);
3737 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3738 ProjNode* proj = fast_out(i)->as_Proj();
3739 if (proj->_con == con) {
3740 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3741 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3742 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3743 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3744 __ cmp(r0, zr);
3745 __ cset(toReg, Assembler::NE);
3746 if (reg->is_stack()) {
3747 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3748 __ str(toReg, Address(sp, st_off));
3749 }
3750 break;
3751 }
3752 }
3753 if (return_value_is_used()) {
3754 // An inline type is returned as fields in multiple registers.
3755 // R0 either contains an oop if the inline type is buffered or a pointer
3756 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3757 // if the lowest bit is set to allow C2 to use the oop after null checking.
3758 // r0 &= (r0 & 1) - 1
3759 __ andr(rscratch1, r0, 0x1);
3760 __ sub(rscratch1, rscratch1, 0x1);
3761 __ andr(r0, r0, rscratch1);
3762 }
3763 }
3764 %}
3765
3766 enc_class aarch64_enc_java_to_runtime(method meth) %{
3767 // some calls to generated routines (arraycopy code) are scheduled
3768 // by C2 as runtime calls. if so we can call them using a br (they
3769 // will be in a reachable segment) otherwise we have to use a blr
3770 // which loads the absolute address into a register.
3771 address entry = (address)$meth$$method;
3772 CodeBlob *cb = CodeCache::find_blob(entry);
3773 if (cb) {
3774 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3775 if (call == nullptr) {
3776 ciEnv::current()->record_failure("CodeCache is full");
3777 return;
3778 }
3779 __ post_call_nop();
3780 } else {
3781 Label retaddr;
3782 // Make the anchor frame walkable
3783 __ adr(rscratch2, retaddr);
6925 instruct loadConL(iRegLNoSp dst, immL src)
6926 %{
6927 match(Set dst src);
6928
6929 ins_cost(INSN_COST);
6930 format %{ "mov $dst, $src\t# long" %}
6931
6932 ins_encode( aarch64_enc_mov_imm(dst, src) );
6933
6934 ins_pipe(ialu_imm);
6935 %}
6936
6937 // Load Pointer Constant
6938
6939 instruct loadConP(iRegPNoSp dst, immP con)
6940 %{
6941 match(Set dst con);
6942
6943 ins_cost(INSN_COST * 4);
6944 format %{
6945 "mov $dst, $con\t# ptr"
6946 %}
6947
6948 ins_encode(aarch64_enc_mov_p(dst, con));
6949
6950 ins_pipe(ialu_imm);
6951 %}
6952
6953 // Load Null Pointer Constant
6954
6955 instruct loadConP0(iRegPNoSp dst, immP0 con)
6956 %{
6957 match(Set dst con);
6958
6959 ins_cost(INSN_COST);
6960 format %{ "mov $dst, $con\t# nullptr ptr" %}
6961
6962 ins_encode(aarch64_enc_mov_p0(dst, con));
6963
6964 ins_pipe(ialu_imm);
6965 %}
8118 %}
8119
8120 // ============================================================================
8121 // Cast/Convert Instructions
8122
8123 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8124 match(Set dst (CastX2P src));
8125
8126 ins_cost(INSN_COST);
8127 format %{ "mov $dst, $src\t# long -> ptr" %}
8128
8129 ins_encode %{
8130 if ($dst$$reg != $src$$reg) {
8131 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8132 }
8133 %}
8134
8135 ins_pipe(ialu_reg);
8136 %}
8137
8138 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8139 match(Set dst (CastI2N src));
8140
8141 ins_cost(INSN_COST);
8142 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8143
8144 ins_encode %{
8145 if ($dst$$reg != $src$$reg) {
8146 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8147 }
8148 %}
8149
8150 ins_pipe(ialu_reg);
8151 %}
8152
8153 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8154 match(Set dst (CastP2X src));
8155
8156 ins_cost(INSN_COST);
8157 format %{ "mov $dst, $src\t# ptr -> long" %}
8158
8159 ins_encode %{
8160 if ($dst$$reg != $src$$reg) {
8161 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8162 }
8163 %}
8164
8165 ins_pipe(ialu_reg);
8166 %}
8167
8168 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8169 match(Set dst (CastP2X src));
8170
8171 ins_cost(INSN_COST);
8172 format %{ "mov $dst, $src\t# ptr -> long" %}
8173
8174 ins_encode %{
8175 if ($dst$$reg != $src$$reg) {
8176 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8177 }
8178 %}
8179
8180 ins_pipe(ialu_reg);
8181 %}
8182
8183 // Convert oop into int for vectors alignment masking
8184 instruct convP2I(iRegINoSp dst, iRegP src) %{
8185 match(Set dst (ConvL2I (CastP2X src)));
8186
8187 ins_cost(INSN_COST);
15101
15102 match(Set dst (MoveL2D src));
15103
15104 effect(DEF dst, USE src);
15105
15106 ins_cost(INSN_COST);
15107
15108 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
15109
15110 ins_encode %{
15111 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
15112 %}
15113
15114 ins_pipe(fp_l2d);
15115
15116 %}
15117
15118 // ============================================================================
15119 // clearing of an array
15120
15121 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
15122 %{
15123 match(Set dummy (ClearArray (Binary cnt base) zero));
15124 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15125
15126 ins_cost(4 * INSN_COST);
15127 format %{ "ClearArray $cnt, $base" %}
15128
15129 ins_encode %{
15130 address tpc = __ zero_words($base$$Register, $cnt$$Register);
15131 if (tpc == nullptr) {
15132 ciEnv::current()->record_failure("CodeCache is full");
15133 return;
15134 }
15135 %}
15136
15137 ins_pipe(pipe_class_memory);
15138 %}
15139
15140 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
15141 %{
15142 predicate(((ClearArrayNode*)n)->word_copy_only());
15143 match(Set dummy (ClearArray (Binary cnt base) val));
15144 effect(USE_KILL cnt, USE_KILL base, KILL cr);
15145
15146 ins_cost(4 * INSN_COST);
15147 format %{ "ClearArray $cnt, $base, $val" %}
15148
15149 ins_encode %{
15150 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
15151 %}
15152
15153 ins_pipe(pipe_class_memory);
15154 %}
15155
15156 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
15157 %{
15158 predicate((uint64_t)n->in(2)->get_long()
15159 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
15160 && !((ClearArrayNode*)n)->word_copy_only());
15161 match(Set dummy (ClearArray cnt base));
15162 effect(TEMP temp, USE_KILL base, KILL cr);
15163
15164 ins_cost(4 * INSN_COST);
15165 format %{ "ClearArray $cnt, $base" %}
15166
15167 ins_encode %{
15168 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
15169 if (tpc == nullptr) {
15170 ciEnv::current()->record_failure("CodeCache is full");
15171 return;
15172 }
15173 %}
15174
15175 ins_pipe(pipe_class_memory);
15176 %}
15177
15178 // ============================================================================
15179 // Overflow Math Instructions
15180
16457 %}
16458
16459 // Call Runtime Instruction without safepoint and with vector arguments
16460 instruct CallLeafDirectVector(method meth)
16461 %{
16462 match(CallLeafVector);
16463
16464 effect(USE meth);
16465
16466 ins_cost(CALL_COST);
16467
16468 format %{ "CALL, runtime leaf vector $meth" %}
16469
16470 ins_encode(aarch64_enc_java_to_runtime(meth));
16471
16472 ins_pipe(pipe_class_call);
16473 %}
16474
16475 // Call Runtime Instruction
16476
16477 // entry point is null, target holds the address to call
16478 instruct CallLeafNoFPIndirect(iRegP target)
16479 %{
16480 predicate(n->as_Call()->entry_point() == nullptr);
16481
16482 match(CallLeafNoFP target);
16483
16484 ins_cost(CALL_COST);
16485
16486 format %{ "CALL, runtime leaf nofp indirect $target" %}
16487
16488 ins_encode %{
16489 __ blr($target$$Register);
16490 %}
16491
16492 ins_pipe(pipe_class_call);
16493 %}
16494
16495 instruct CallLeafNoFPDirect(method meth)
16496 %{
16497 predicate(n->as_Call()->entry_point() != nullptr);
16498
16499 match(CallLeafNoFP);
16500
16501 effect(USE meth);
16502
16503 ins_cost(CALL_COST);
16504
16505 format %{ "CALL, runtime leaf nofp $meth" %}
16506
16507 ins_encode( aarch64_enc_java_to_runtime(meth) );
16508
16509 ins_pipe(pipe_class_call);
16510 %}
16511
16512 // Tail Call; Jump from runtime stub to Java code.
16513 // Also known as an 'interprocedural jump'.
16514 // Target of jump will eventually return to caller.
16515 // TailJump below removes the return address.
16516 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
16517 // emitted just above the TailCall which has reset rfp to the caller state.
16518 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
|