1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else {
1694 return 6 * NativeInstruction::instruction_size;
1695 }
1696 }
1697
1698 //=============================================================================
1699
1700 #ifndef PRODUCT
1701 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1702 st->print("BREAKPOINT");
1703 }
1704 #endif
1705
1706 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1707 __ brk(0);
1708 }
1709
1710 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1711 return MachNode::size(ra_);
1712 }
1781 if (C->stub_function() == nullptr) {
1782 st->print("\n\t");
1783 st->print("ldr rscratch1, [guard]\n\t");
1784 st->print("dmb ishld\n\t");
1785 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1786 st->print("cmp rscratch1, rscratch2\n\t");
1787 st->print("b.eq skip");
1788 st->print("\n\t");
1789 st->print("blr #nmethod_entry_barrier_stub\n\t");
1790 st->print("b skip\n\t");
1791 st->print("guard: int\n\t");
1792 st->print("\n\t");
1793 st->print("skip:\n\t");
1794 }
1795 }
1796 #endif
1797
1798 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1799 Compile* C = ra_->C;
1800
1801 // n.b. frame size includes space for return pc and rfp
1802 const int framesize = C->output()->frame_size_in_bytes();
1803
1804 if (C->clinit_barrier_on_entry()) {
1805 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1806
1807 Label L_skip_barrier;
1808
1809 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1810 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1811 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1812 __ bind(L_skip_barrier);
1813 }
1814
1815 if (C->max_vector_size() > 0) {
1816 __ reinitialize_ptrue();
1817 }
1818
1819 int bangsize = C->output()->bang_size_in_bytes();
1820 if (C->output()->need_stack_bang(bangsize))
1821 __ generate_stack_overflow_check(bangsize);
1822
1823 __ build_frame(framesize);
1824
1825 if (C->stub_function() == nullptr) {
1826 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1827 // Dummy labels for just measuring the code size
1828 Label dummy_slow_path;
1829 Label dummy_continuation;
1830 Label dummy_guard;
1831 Label* slow_path = &dummy_slow_path;
1832 Label* continuation = &dummy_continuation;
1833 Label* guard = &dummy_guard;
1834 if (!Compile::current()->output()->in_scratch_emit_size()) {
1835 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1836 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1837 Compile::current()->output()->add_stub(stub);
1838 slow_path = &stub->entry();
1839 continuation = &stub->continuation();
1840 guard = &stub->guard();
1841 }
1842 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1843 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1844 }
1845
1846 if (VerifyStackAtCalls) {
1847 Unimplemented();
1848 }
1849
1850 C->output()->set_frame_complete(__ offset());
1851
1852 if (C->has_mach_constant_base_node()) {
1853 // NOTE: We set the table base offset here because users might be
1854 // emitted before MachConstantBaseNode.
1855 ConstantTable& constant_table = C->output()->constant_table();
1856 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1857 }
1858 }
1859
1860 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1861 {
1862 return MachNode::size(ra_); // too many variables; just compute it
1863 // the hard way
1864 }
1865
1866 int MachPrologNode::reloc() const
1867 {
1868 return 0;
1869 }
1870
1871 //=============================================================================
1872
1873 #ifndef PRODUCT
1874 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1875 Compile* C = ra_->C;
1876 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1877
1878 st->print("# pop frame %d\n\t",framesize);
1879
1880 if (framesize == 0) {
1881 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1882 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1883 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1884 st->print("add sp, sp, #%d\n\t", framesize);
1885 } else {
1888 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1889 }
1890 if (VM_Version::use_rop_protection()) {
1891 st->print("autiaz\n\t");
1892 st->print("ldr zr, [lr]\n\t");
1893 }
1894
1895 if (do_polling() && C->is_method_compilation()) {
1896 st->print("# test polling word\n\t");
1897 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1898 st->print("cmp sp, rscratch1\n\t");
1899 st->print("bhi #slow_path");
1900 }
1901 }
1902 #endif
1903
1904 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1905 Compile* C = ra_->C;
1906 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1907
1908 __ remove_frame(framesize);
1909
1910 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1911 __ reserved_stack_check();
1912 }
1913
1914 if (do_polling() && C->is_method_compilation()) {
1915 Label dummy_label;
1916 Label* code_stub = &dummy_label;
1917 if (!C->output()->in_scratch_emit_size()) {
1918 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1919 C->output()->add_stub(stub);
1920 code_stub = &stub->entry();
1921 }
1922 __ relocate(relocInfo::poll_return_type);
1923 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1924 }
1925 }
1926
1927 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1928 // Variable size. Determine dynamically.
1929 return MachNode::size(ra_);
1930 }
1931
1932 int MachEpilogNode::reloc() const {
1933 // Return number of relocatable values contained in this instruction.
1934 return 1; // 1 for polling page.
1935 }
1936
1937 const Pipeline * MachEpilogNode::pipeline() const {
1938 return MachNode::pipeline_class();
1939 }
1940
1941 //=============================================================================
1942
1943 static enum RC rc_class(OptoReg::Name reg) {
1944
1945 if (reg == OptoReg::Bad) {
1946 return rc_bad;
1947 }
1948
1949 // we have 32 int registers * 2 halves
1950 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1951
2210 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2211 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2212 int reg = ra_->get_encode(this);
2213
2214 // This add will handle any 24-bit signed offset. 24 bits allows an
2215 // 8 megabyte stack frame.
2216 __ add(as_Register(reg), sp, offset);
2217 }
2218
2219 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2220 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2221 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2222
2223 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2224 return NativeInstruction::instruction_size;
2225 } else {
2226 return 2 * NativeInstruction::instruction_size;
2227 }
2228 }
2229
2230 //=============================================================================
2231
2232 #ifndef PRODUCT
2233 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2234 {
2235 st->print_cr("# MachUEPNode");
2236 if (UseCompressedClassPointers) {
2237 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2238 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2239 st->print_cr("\tcmpw rscratch1, r10");
2240 } else {
2241 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2242 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2243 st->print_cr("\tcmp rscratch1, r10");
2244 }
2245 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2246 }
2247 #endif
2248
2249 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2250 {
2251 __ ic_check(InteriorEntryAlignment);
2252 }
2253
2254 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2255 {
2256 return MachNode::size(ra_);
2257 }
2258
2259 // REQUIRED EMIT CODE
2260
2261 //=============================================================================
2262
2263 // Emit deopt handler code.
2264 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2265 {
2266 // Note that the code buffer's insts_mark is always relative to insts.
2267 // That's why we must use the macroassembler to generate a handler.
2268 address base = __ start_a_stub(size_deopt_handler());
2269 if (base == nullptr) {
2270 ciEnv::current()->record_failure("CodeCache is full");
2271 return 0; // CodeBuffer::expand failed
2272 }
2273
2274 int offset = __ offset();
2275 Label start;
2276 __ bind(start);
2277 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2278
3659 %}
3660
3661 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3662 int method_index = resolved_method_index(masm);
3663 address call = __ ic_call((address)$meth$$method, method_index);
3664 if (call == nullptr) {
3665 ciEnv::current()->record_failure("CodeCache is full");
3666 return;
3667 }
3668 __ post_call_nop();
3669 if (Compile::current()->max_vector_size() > 0) {
3670 __ reinitialize_ptrue();
3671 }
3672 %}
3673
3674 enc_class aarch64_enc_call_epilog() %{
3675 if (VerifyStackAtCalls) {
3676 // Check that stack depth is unchanged: find majik cookie on stack
3677 __ call_Unimplemented();
3678 }
3679 %}
3680
3681 enc_class aarch64_enc_java_to_runtime(method meth) %{
3682 // some calls to generated routines (arraycopy code) are scheduled
3683 // by C2 as runtime calls. if so we can call them using a br (they
3684 // will be in a reachable segment) otherwise we have to use a blr
3685 // which loads the absolute address into a register.
3686 address entry = (address)$meth$$method;
3687 CodeBlob *cb = CodeCache::find_blob(entry);
3688 if (cb) {
3689 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3690 if (call == nullptr) {
3691 ciEnv::current()->record_failure("CodeCache is full");
3692 return;
3693 }
3694 __ post_call_nop();
3695 } else {
3696 Label retaddr;
3697 // Make the anchor frame walkable
3698 __ adr(rscratch2, retaddr);
3948 operand immI_gt_1()
3949 %{
3950 predicate(n->get_int() > 1);
3951 match(ConI);
3952
3953 op_cost(0);
3954 format %{ %}
3955 interface(CONST_INTER);
3956 %}
3957
3958 operand immI_le_4()
3959 %{
3960 predicate(n->get_int() <= 4);
3961 match(ConI);
3962
3963 op_cost(0);
3964 format %{ %}
3965 interface(CONST_INTER);
3966 %}
3967
3968 operand immI_16()
3969 %{
3970 predicate(n->get_int() == 16);
3971 match(ConI);
3972
3973 op_cost(0);
3974 format %{ %}
3975 interface(CONST_INTER);
3976 %}
3977
3978 operand immI_24()
3979 %{
3980 predicate(n->get_int() == 24);
3981 match(ConI);
3982
3983 op_cost(0);
3984 format %{ %}
3985 interface(CONST_INTER);
3986 %}
3987
6847 instruct loadConL(iRegLNoSp dst, immL src)
6848 %{
6849 match(Set dst src);
6850
6851 ins_cost(INSN_COST);
6852 format %{ "mov $dst, $src\t# long" %}
6853
6854 ins_encode( aarch64_enc_mov_imm(dst, src) );
6855
6856 ins_pipe(ialu_imm);
6857 %}
6858
6859 // Load Pointer Constant
6860
6861 instruct loadConP(iRegPNoSp dst, immP con)
6862 %{
6863 match(Set dst con);
6864
6865 ins_cost(INSN_COST * 4);
6866 format %{
6867 "mov $dst, $con\t# ptr\n\t"
6868 %}
6869
6870 ins_encode(aarch64_enc_mov_p(dst, con));
6871
6872 ins_pipe(ialu_imm);
6873 %}
6874
6875 // Load Null Pointer Constant
6876
6877 instruct loadConP0(iRegPNoSp dst, immP0 con)
6878 %{
6879 match(Set dst con);
6880
6881 ins_cost(INSN_COST);
6882 format %{ "mov $dst, $con\t# nullptr ptr" %}
6883
6884 ins_encode(aarch64_enc_mov_p0(dst, con));
6885
6886 ins_pipe(ialu_imm);
6887 %}
8054 %}
8055
8056 // ============================================================================
8057 // Cast/Convert Instructions
8058
8059 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8060 match(Set dst (CastX2P src));
8061
8062 ins_cost(INSN_COST);
8063 format %{ "mov $dst, $src\t# long -> ptr" %}
8064
8065 ins_encode %{
8066 if ($dst$$reg != $src$$reg) {
8067 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8068 }
8069 %}
8070
8071 ins_pipe(ialu_reg);
8072 %}
8073
8074 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8075 match(Set dst (CastP2X src));
8076
8077 ins_cost(INSN_COST);
8078 format %{ "mov $dst, $src\t# ptr -> long" %}
8079
8080 ins_encode %{
8081 if ($dst$$reg != $src$$reg) {
8082 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8083 }
8084 %}
8085
8086 ins_pipe(ialu_reg);
8087 %}
8088
8089 // Convert oop into int for vectors alignment masking
8090 instruct convP2I(iRegINoSp dst, iRegP src) %{
8091 match(Set dst (ConvL2I (CastP2X src)));
8092
8093 ins_cost(INSN_COST);
14042
14043 match(Set dst (MoveL2D src));
14044
14045 effect(DEF dst, USE src);
14046
14047 ins_cost(INSN_COST);
14048
14049 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14050
14051 ins_encode %{
14052 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14053 %}
14054
14055 ins_pipe(fp_l2d);
14056
14057 %}
14058
14059 // ============================================================================
14060 // clearing of an array
14061
14062 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14063 %{
14064 match(Set dummy (ClearArray cnt base));
14065 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14066
14067 ins_cost(4 * INSN_COST);
14068 format %{ "ClearArray $cnt, $base" %}
14069
14070 ins_encode %{
14071 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14072 if (tpc == nullptr) {
14073 ciEnv::current()->record_failure("CodeCache is full");
14074 return;
14075 }
14076 %}
14077
14078 ins_pipe(pipe_class_memory);
14079 %}
14080
14081 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14082 %{
14083 predicate((uint64_t)n->in(2)->get_long()
14084 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14085 match(Set dummy (ClearArray cnt base));
14086 effect(TEMP temp, USE_KILL base, KILL cr);
14087
14088 ins_cost(4 * INSN_COST);
14089 format %{ "ClearArray $cnt, $base" %}
14090
14091 ins_encode %{
14092 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14093 if (tpc == nullptr) {
14094 ciEnv::current()->record_failure("CodeCache is full");
14095 return;
14096 }
14097 %}
14098
14099 ins_pipe(pipe_class_memory);
14100 %}
14101
14102 // ============================================================================
14103 // Overflow Math Instructions
14104
15381 %}
15382
15383 // Call Runtime Instruction without safepoint and with vector arguments
15384 instruct CallLeafDirectVector(method meth)
15385 %{
15386 match(CallLeafVector);
15387
15388 effect(USE meth);
15389
15390 ins_cost(CALL_COST);
15391
15392 format %{ "CALL, runtime leaf vector $meth" %}
15393
15394 ins_encode(aarch64_enc_java_to_runtime(meth));
15395
15396 ins_pipe(pipe_class_call);
15397 %}
15398
15399 // Call Runtime Instruction
15400
15401 instruct CallLeafNoFPDirect(method meth)
15402 %{
15403 match(CallLeafNoFP);
15404
15405 effect(USE meth);
15406
15407 ins_cost(CALL_COST);
15408
15409 format %{ "CALL, runtime leaf nofp $meth" %}
15410
15411 ins_encode( aarch64_enc_java_to_runtime(meth) );
15412
15413 ins_pipe(pipe_class_call);
15414 %}
15415
15416 // Tail Call; Jump from runtime stub to Java code.
15417 // Also known as an 'interprocedural jump'.
15418 // Target of jump will eventually return to caller.
15419 // TailJump below removes the return address.
15420 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15421 // emitted just above the TailCall which has reset rfp to the caller state.
15422 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
|
1673
1674 int MachCallDynamicJavaNode::ret_addr_offset()
1675 {
1676 return 16; // movz, movk, movk, bl
1677 }
1678
1679 int MachCallRuntimeNode::ret_addr_offset() {
1680 // for generated stubs the call will be
1681 // bl(addr)
1682 // or with far branches
1683 // bl(trampoline_stub)
1684 // for real runtime callouts it will be six instructions
1685 // see aarch64_enc_java_to_runtime
1686 // adr(rscratch2, retaddr)
1687 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1688 // lea(rscratch1, RuntimeAddress(addr)
1689 // blr(rscratch1)
1690 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1691 if (cb) {
1692 return 1 * NativeInstruction::instruction_size;
1693 } else if (_entry_point == nullptr) {
1694 // See CallLeafNoFPIndirect
1695 return 1 * NativeInstruction::instruction_size;
1696 } else {
1697 return 6 * NativeInstruction::instruction_size;
1698 }
1699 }
1700
1701 //=============================================================================
1702
1703 #ifndef PRODUCT
1704 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1705 st->print("BREAKPOINT");
1706 }
1707 #endif
1708
1709 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1710 __ brk(0);
1711 }
1712
1713 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1714 return MachNode::size(ra_);
1715 }
1784 if (C->stub_function() == nullptr) {
1785 st->print("\n\t");
1786 st->print("ldr rscratch1, [guard]\n\t");
1787 st->print("dmb ishld\n\t");
1788 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1789 st->print("cmp rscratch1, rscratch2\n\t");
1790 st->print("b.eq skip");
1791 st->print("\n\t");
1792 st->print("blr #nmethod_entry_barrier_stub\n\t");
1793 st->print("b skip\n\t");
1794 st->print("guard: int\n\t");
1795 st->print("\n\t");
1796 st->print("skip:\n\t");
1797 }
1798 }
1799 #endif
1800
1801 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1802 Compile* C = ra_->C;
1803
1804
1805 __ verified_entry(C, 0);
1806
1807 if (C->stub_function() == nullptr) {
1808 __ entry_barrier();
1809 }
1810
1811 if (!Compile::current()->output()->in_scratch_emit_size()) {
1812 __ bind(*_verified_entry);
1813 }
1814
1815 if (VerifyStackAtCalls) {
1816 Unimplemented();
1817 }
1818
1819 C->output()->set_frame_complete(__ offset());
1820
1821 if (C->has_mach_constant_base_node()) {
1822 // NOTE: We set the table base offset here because users might be
1823 // emitted before MachConstantBaseNode.
1824 ConstantTable& constant_table = C->output()->constant_table();
1825 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1826 }
1827 }
1828
1829 int MachPrologNode::reloc() const
1830 {
1831 return 0;
1832 }
1833
1834 //=============================================================================
1835
1836 #ifndef PRODUCT
1837 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1838 Compile* C = ra_->C;
1839 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1840
1841 st->print("# pop frame %d\n\t",framesize);
1842
1843 if (framesize == 0) {
1844 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1845 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1846 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1847 st->print("add sp, sp, #%d\n\t", framesize);
1848 } else {
1851 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1852 }
1853 if (VM_Version::use_rop_protection()) {
1854 st->print("autiaz\n\t");
1855 st->print("ldr zr, [lr]\n\t");
1856 }
1857
1858 if (do_polling() && C->is_method_compilation()) {
1859 st->print("# test polling word\n\t");
1860 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1861 st->print("cmp sp, rscratch1\n\t");
1862 st->print("bhi #slow_path");
1863 }
1864 }
1865 #endif
1866
1867 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1868 Compile* C = ra_->C;
1869 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1870
1871 __ remove_frame(framesize, C->needs_stack_repair());
1872
1873 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1874 __ reserved_stack_check();
1875 }
1876
1877 if (do_polling() && C->is_method_compilation()) {
1878 Label dummy_label;
1879 Label* code_stub = &dummy_label;
1880 if (!C->output()->in_scratch_emit_size()) {
1881 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1882 C->output()->add_stub(stub);
1883 code_stub = &stub->entry();
1884 }
1885 __ relocate(relocInfo::poll_return_type);
1886 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1887 }
1888 }
1889
1890 int MachEpilogNode::reloc() const {
1891 // Return number of relocatable values contained in this instruction.
1892 return 1; // 1 for polling page.
1893 }
1894
1895 const Pipeline * MachEpilogNode::pipeline() const {
1896 return MachNode::pipeline_class();
1897 }
1898
1899 //=============================================================================
1900
1901 static enum RC rc_class(OptoReg::Name reg) {
1902
1903 if (reg == OptoReg::Bad) {
1904 return rc_bad;
1905 }
1906
1907 // we have 32 int registers * 2 halves
1908 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1909
2168 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2169 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2170 int reg = ra_->get_encode(this);
2171
2172 // This add will handle any 24-bit signed offset. 24 bits allows an
2173 // 8 megabyte stack frame.
2174 __ add(as_Register(reg), sp, offset);
2175 }
2176
2177 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2178 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2179 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2180
2181 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2182 return NativeInstruction::instruction_size;
2183 } else {
2184 return 2 * NativeInstruction::instruction_size;
2185 }
2186 }
2187
2188 ///=============================================================================
2189 #ifndef PRODUCT
2190 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2191 {
2192 st->print_cr("# MachVEPNode");
2193 if (!_verified) {
2194 st->print_cr("\t load_class");
2195 } else {
2196 st->print_cr("\t unpack_inline_arg");
2197 }
2198 }
2199 #endif
2200
2201 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2202 {
2203 if (!_verified) {
2204 __ ic_check(1);
2205 } else {
2206 // TODO 8284443 Avoid creation of temporary frame
2207 if (ra_->C->stub_function() == nullptr) {
2208 __ verified_entry(ra_->C, 0);
2209 __ entry_barrier();
2210 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2211 __ remove_frame(framesize, false);
2212 }
2213 // Unpack inline type args passed as oop and then jump to
2214 // the verified entry point (skipping the unverified entry).
2215 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2216 // Emit code for verified entry and save increment for stack repair on return
2217 __ verified_entry(ra_->C, sp_inc);
2218 if (Compile::current()->output()->in_scratch_emit_size()) {
2219 Label dummy_verified_entry;
2220 __ b(dummy_verified_entry);
2221 } else {
2222 __ b(*_verified_entry);
2223 }
2224 }
2225 }
2226
2227 //=============================================================================
2228 #ifndef PRODUCT
2229 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2230 {
2231 st->print_cr("# MachUEPNode");
2232 if (UseCompressedClassPointers) {
2233 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2234 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2235 st->print_cr("\tcmpw rscratch1, r10");
2236 } else {
2237 st->print_cr("\tldr rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2238 st->print_cr("\tldr r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2239 st->print_cr("\tcmp rscratch1, r10");
2240 }
2241 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2242 }
2243 #endif
2244
2245 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2246 {
2247 __ ic_check(InteriorEntryAlignment);
2248 }
2249
2250 // REQUIRED EMIT CODE
2251
2252 //=============================================================================
2253
2254 // Emit deopt handler code.
2255 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2256 {
2257 // Note that the code buffer's insts_mark is always relative to insts.
2258 // That's why we must use the macroassembler to generate a handler.
2259 address base = __ start_a_stub(size_deopt_handler());
2260 if (base == nullptr) {
2261 ciEnv::current()->record_failure("CodeCache is full");
2262 return 0; // CodeBuffer::expand failed
2263 }
2264
2265 int offset = __ offset();
2266 Label start;
2267 __ bind(start);
2268 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2269
3650 %}
3651
3652 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3653 int method_index = resolved_method_index(masm);
3654 address call = __ ic_call((address)$meth$$method, method_index);
3655 if (call == nullptr) {
3656 ciEnv::current()->record_failure("CodeCache is full");
3657 return;
3658 }
3659 __ post_call_nop();
3660 if (Compile::current()->max_vector_size() > 0) {
3661 __ reinitialize_ptrue();
3662 }
3663 %}
3664
3665 enc_class aarch64_enc_call_epilog() %{
3666 if (VerifyStackAtCalls) {
3667 // Check that stack depth is unchanged: find majik cookie on stack
3668 __ call_Unimplemented();
3669 }
3670 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3671 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3672 // Search for the corresponding projection, get the register and emit code that initialized it.
3673 uint con = (tf()->range_cc()->cnt() - 1);
3674 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3675 ProjNode* proj = fast_out(i)->as_Proj();
3676 if (proj->_con == con) {
3677 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3678 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3679 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3680 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3681 __ cmp(r0, zr);
3682 __ cset(toReg, Assembler::NE);
3683 if (reg->is_stack()) {
3684 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3685 __ str(toReg, Address(sp, st_off));
3686 }
3687 break;
3688 }
3689 }
3690 if (return_value_is_used()) {
3691 // An inline type is returned as fields in multiple registers.
3692 // R0 either contains an oop if the inline type is buffered or a pointer
3693 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3694 // if the lowest bit is set to allow C2 to use the oop after null checking.
3695 // r0 &= (r0 & 1) - 1
3696 __ andr(rscratch1, r0, 0x1);
3697 __ sub(rscratch1, rscratch1, 0x1);
3698 __ andr(r0, r0, rscratch1);
3699 }
3700 }
3701 %}
3702
3703 enc_class aarch64_enc_java_to_runtime(method meth) %{
3704 // some calls to generated routines (arraycopy code) are scheduled
3705 // by C2 as runtime calls. if so we can call them using a br (they
3706 // will be in a reachable segment) otherwise we have to use a blr
3707 // which loads the absolute address into a register.
3708 address entry = (address)$meth$$method;
3709 CodeBlob *cb = CodeCache::find_blob(entry);
3710 if (cb) {
3711 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3712 if (call == nullptr) {
3713 ciEnv::current()->record_failure("CodeCache is full");
3714 return;
3715 }
3716 __ post_call_nop();
3717 } else {
3718 Label retaddr;
3719 // Make the anchor frame walkable
3720 __ adr(rscratch2, retaddr);
3970 operand immI_gt_1()
3971 %{
3972 predicate(n->get_int() > 1);
3973 match(ConI);
3974
3975 op_cost(0);
3976 format %{ %}
3977 interface(CONST_INTER);
3978 %}
3979
3980 operand immI_le_4()
3981 %{
3982 predicate(n->get_int() <= 4);
3983 match(ConI);
3984
3985 op_cost(0);
3986 format %{ %}
3987 interface(CONST_INTER);
3988 %}
3989
3990 operand immI_4()
3991 %{
3992 predicate(n->get_int() == 4);
3993 match(ConI);
3994
3995 op_cost(0);
3996 format %{ %}
3997 interface(CONST_INTER);
3998 %}
3999
4000 operand immI_16()
4001 %{
4002 predicate(n->get_int() == 16);
4003 match(ConI);
4004
4005 op_cost(0);
4006 format %{ %}
4007 interface(CONST_INTER);
4008 %}
4009
4010 operand immI_24()
4011 %{
4012 predicate(n->get_int() == 24);
4013 match(ConI);
4014
4015 op_cost(0);
4016 format %{ %}
4017 interface(CONST_INTER);
4018 %}
4019
6879 instruct loadConL(iRegLNoSp dst, immL src)
6880 %{
6881 match(Set dst src);
6882
6883 ins_cost(INSN_COST);
6884 format %{ "mov $dst, $src\t# long" %}
6885
6886 ins_encode( aarch64_enc_mov_imm(dst, src) );
6887
6888 ins_pipe(ialu_imm);
6889 %}
6890
6891 // Load Pointer Constant
6892
6893 instruct loadConP(iRegPNoSp dst, immP con)
6894 %{
6895 match(Set dst con);
6896
6897 ins_cost(INSN_COST * 4);
6898 format %{
6899 "mov $dst, $con\t# ptr"
6900 %}
6901
6902 ins_encode(aarch64_enc_mov_p(dst, con));
6903
6904 ins_pipe(ialu_imm);
6905 %}
6906
6907 // Load Null Pointer Constant
6908
6909 instruct loadConP0(iRegPNoSp dst, immP0 con)
6910 %{
6911 match(Set dst con);
6912
6913 ins_cost(INSN_COST);
6914 format %{ "mov $dst, $con\t# nullptr ptr" %}
6915
6916 ins_encode(aarch64_enc_mov_p0(dst, con));
6917
6918 ins_pipe(ialu_imm);
6919 %}
8086 %}
8087
8088 // ============================================================================
8089 // Cast/Convert Instructions
8090
8091 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8092 match(Set dst (CastX2P src));
8093
8094 ins_cost(INSN_COST);
8095 format %{ "mov $dst, $src\t# long -> ptr" %}
8096
8097 ins_encode %{
8098 if ($dst$$reg != $src$$reg) {
8099 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8100 }
8101 %}
8102
8103 ins_pipe(ialu_reg);
8104 %}
8105
8106 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8107 match(Set dst (CastI2N src));
8108
8109 ins_cost(INSN_COST);
8110 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8111
8112 ins_encode %{
8113 if ($dst$$reg != $src$$reg) {
8114 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8115 }
8116 %}
8117
8118 ins_pipe(ialu_reg);
8119 %}
8120
8121 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8122 match(Set dst (CastP2X src));
8123
8124 ins_cost(INSN_COST);
8125 format %{ "mov $dst, $src\t# ptr -> long" %}
8126
8127 ins_encode %{
8128 if ($dst$$reg != $src$$reg) {
8129 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8130 }
8131 %}
8132
8133 ins_pipe(ialu_reg);
8134 %}
8135
8136 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8137 match(Set dst (CastP2X src));
8138
8139 ins_cost(INSN_COST);
8140 format %{ "mov $dst, $src\t# ptr -> long" %}
8141
8142 ins_encode %{
8143 if ($dst$$reg != $src$$reg) {
8144 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8145 }
8146 %}
8147
8148 ins_pipe(ialu_reg);
8149 %}
8150
8151 // Convert oop into int for vectors alignment masking
8152 instruct convP2I(iRegINoSp dst, iRegP src) %{
8153 match(Set dst (ConvL2I (CastP2X src)));
8154
8155 ins_cost(INSN_COST);
14104
14105 match(Set dst (MoveL2D src));
14106
14107 effect(DEF dst, USE src);
14108
14109 ins_cost(INSN_COST);
14110
14111 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14112
14113 ins_encode %{
14114 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14115 %}
14116
14117 ins_pipe(fp_l2d);
14118
14119 %}
14120
14121 // ============================================================================
14122 // clearing of an array
14123
14124 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14125 %{
14126 match(Set dummy (ClearArray (Binary cnt base) zero));
14127 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14128
14129 ins_cost(4 * INSN_COST);
14130 format %{ "ClearArray $cnt, $base" %}
14131
14132 ins_encode %{
14133 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14134 if (tpc == nullptr) {
14135 ciEnv::current()->record_failure("CodeCache is full");
14136 return;
14137 }
14138 %}
14139
14140 ins_pipe(pipe_class_memory);
14141 %}
14142
14143 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14144 %{
14145 predicate(((ClearArrayNode*)n)->word_copy_only());
14146 match(Set dummy (ClearArray (Binary cnt base) val));
14147 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14148
14149 ins_cost(4 * INSN_COST);
14150 format %{ "ClearArray $cnt, $base, $val" %}
14151
14152 ins_encode %{
14153 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14154 %}
14155
14156 ins_pipe(pipe_class_memory);
14157 %}
14158
14159 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14160 %{
14161 predicate((uint64_t)n->in(2)->get_long()
14162 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14163 && !((ClearArrayNode*)n)->word_copy_only());
14164 match(Set dummy (ClearArray cnt base));
14165 effect(TEMP temp, USE_KILL base, KILL cr);
14166
14167 ins_cost(4 * INSN_COST);
14168 format %{ "ClearArray $cnt, $base" %}
14169
14170 ins_encode %{
14171 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14172 if (tpc == nullptr) {
14173 ciEnv::current()->record_failure("CodeCache is full");
14174 return;
14175 }
14176 %}
14177
14178 ins_pipe(pipe_class_memory);
14179 %}
14180
14181 // ============================================================================
14182 // Overflow Math Instructions
14183
15460 %}
15461
15462 // Call Runtime Instruction without safepoint and with vector arguments
15463 instruct CallLeafDirectVector(method meth)
15464 %{
15465 match(CallLeafVector);
15466
15467 effect(USE meth);
15468
15469 ins_cost(CALL_COST);
15470
15471 format %{ "CALL, runtime leaf vector $meth" %}
15472
15473 ins_encode(aarch64_enc_java_to_runtime(meth));
15474
15475 ins_pipe(pipe_class_call);
15476 %}
15477
15478 // Call Runtime Instruction
15479
15480 // entry point is null, target holds the address to call
15481 instruct CallLeafNoFPIndirect(iRegP target)
15482 %{
15483 predicate(n->as_Call()->entry_point() == nullptr);
15484
15485 match(CallLeafNoFP target);
15486
15487 ins_cost(CALL_COST);
15488
15489 format %{ "CALL, runtime leaf nofp indirect $target" %}
15490
15491 ins_encode %{
15492 __ blr($target$$Register);
15493 %}
15494
15495 ins_pipe(pipe_class_call);
15496 %}
15497
15498 instruct CallLeafNoFPDirect(method meth)
15499 %{
15500 predicate(n->as_Call()->entry_point() != nullptr);
15501
15502 match(CallLeafNoFP);
15503
15504 effect(USE meth);
15505
15506 ins_cost(CALL_COST);
15507
15508 format %{ "CALL, runtime leaf nofp $meth" %}
15509
15510 ins_encode( aarch64_enc_java_to_runtime(meth) );
15511
15512 ins_pipe(pipe_class_call);
15513 %}
15514
15515 // Tail Call; Jump from runtime stub to Java code.
15516 // Also known as an 'interprocedural jump'.
15517 // Target of jump will eventually return to caller.
15518 // TailJump below removes the return address.
15519 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15520 // emitted just above the TailCall which has reset rfp to the caller state.
15521 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
|