1671
1672 int MachCallDynamicJavaNode::ret_addr_offset()
1673 {
1674 return 16; // movz, movk, movk, bl
1675 }
1676
1677 int MachCallRuntimeNode::ret_addr_offset() {
1678 // for generated stubs the call will be
1679 // bl(addr)
1680 // or with far branches
1681 // bl(trampoline_stub)
1682 // for real runtime callouts it will be six instructions
1683 // see aarch64_enc_java_to_runtime
1684 // adr(rscratch2, retaddr)
1685 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1686 // lea(rscratch1, RuntimeAddress(addr)
1687 // blr(rscratch1)
1688 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1689 if (cb) {
1690 return 1 * NativeInstruction::instruction_size;
1691 } else {
1692 return 6 * NativeInstruction::instruction_size;
1693 }
1694 }
1695
1696 //=============================================================================
1697
1698 #ifndef PRODUCT
1699 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1700 st->print("BREAKPOINT");
1701 }
1702 #endif
1703
1704 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1705 __ brk(0);
1706 }
1707
1708 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1709 return MachNode::size(ra_);
1710 }
1779 if (C->stub_function() == nullptr) {
1780 st->print("\n\t");
1781 st->print("ldr rscratch1, [guard]\n\t");
1782 st->print("dmb ishld\n\t");
1783 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1784 st->print("cmp rscratch1, rscratch2\n\t");
1785 st->print("b.eq skip");
1786 st->print("\n\t");
1787 st->print("blr #nmethod_entry_barrier_stub\n\t");
1788 st->print("b skip\n\t");
1789 st->print("guard: int\n\t");
1790 st->print("\n\t");
1791 st->print("skip:\n\t");
1792 }
1793 }
1794 #endif
1795
1796 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1797 Compile* C = ra_->C;
1798
1799 // n.b. frame size includes space for return pc and rfp
1800 const int framesize = C->output()->frame_size_in_bytes();
1801
1802 if (C->clinit_barrier_on_entry()) {
1803 assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
1804
1805 Label L_skip_barrier;
1806
1807 __ mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
1808 __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1809 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1810 __ bind(L_skip_barrier);
1811 }
1812
1813 if (C->max_vector_size() > 0) {
1814 __ reinitialize_ptrue();
1815 }
1816
1817 int bangsize = C->output()->bang_size_in_bytes();
1818 if (C->output()->need_stack_bang(bangsize))
1819 __ generate_stack_overflow_check(bangsize);
1820
1821 __ build_frame(framesize);
1822
1823 if (C->stub_function() == nullptr) {
1824 BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1825 // Dummy labels for just measuring the code size
1826 Label dummy_slow_path;
1827 Label dummy_continuation;
1828 Label dummy_guard;
1829 Label* slow_path = &dummy_slow_path;
1830 Label* continuation = &dummy_continuation;
1831 Label* guard = &dummy_guard;
1832 if (!Compile::current()->output()->in_scratch_emit_size()) {
1833 // Use real labels from actual stub when not emitting code for the purpose of measuring its size
1834 C2EntryBarrierStub* stub = new (Compile::current()->comp_arena()) C2EntryBarrierStub();
1835 Compile::current()->output()->add_stub(stub);
1836 slow_path = &stub->entry();
1837 continuation = &stub->continuation();
1838 guard = &stub->guard();
1839 }
1840 // In the C2 code, we move the non-hot part of nmethod entry barriers out-of-line to a stub.
1841 bs->nmethod_entry_barrier(masm, slow_path, continuation, guard);
1842 }
1843
1844 if (VerifyStackAtCalls) {
1845 Unimplemented();
1846 }
1847
1848 C->output()->set_frame_complete(__ offset());
1849
1850 if (C->has_mach_constant_base_node()) {
1851 // NOTE: We set the table base offset here because users might be
1852 // emitted before MachConstantBaseNode.
1853 ConstantTable& constant_table = C->output()->constant_table();
1854 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1855 }
1856 }
1857
1858 uint MachPrologNode::size(PhaseRegAlloc* ra_) const
1859 {
1860 return MachNode::size(ra_); // too many variables; just compute it
1861 // the hard way
1862 }
1863
1864 int MachPrologNode::reloc() const
1865 {
1866 return 0;
1867 }
1868
1869 //=============================================================================
1870
1871 #ifndef PRODUCT
1872 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1873 Compile* C = ra_->C;
1874 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1875
1876 st->print("# pop frame %d\n\t",framesize);
1877
1878 if (framesize == 0) {
1879 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1880 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1881 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1882 st->print("add sp, sp, #%d\n\t", framesize);
1883 } else {
1886 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1887 }
1888 if (VM_Version::use_rop_protection()) {
1889 st->print("autiaz\n\t");
1890 st->print("ldr zr, [lr]\n\t");
1891 }
1892
1893 if (do_polling() && C->is_method_compilation()) {
1894 st->print("# test polling word\n\t");
1895 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1896 st->print("cmp sp, rscratch1\n\t");
1897 st->print("bhi #slow_path");
1898 }
1899 }
1900 #endif
1901
1902 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1903 Compile* C = ra_->C;
1904 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1905
1906 __ remove_frame(framesize);
1907
1908 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1909 __ reserved_stack_check();
1910 }
1911
1912 if (do_polling() && C->is_method_compilation()) {
1913 Label dummy_label;
1914 Label* code_stub = &dummy_label;
1915 if (!C->output()->in_scratch_emit_size()) {
1916 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1917 C->output()->add_stub(stub);
1918 code_stub = &stub->entry();
1919 }
1920 __ relocate(relocInfo::poll_return_type);
1921 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1922 }
1923 }
1924
1925 uint MachEpilogNode::size(PhaseRegAlloc *ra_) const {
1926 // Variable size. Determine dynamically.
1927 return MachNode::size(ra_);
1928 }
1929
1930 int MachEpilogNode::reloc() const {
1931 // Return number of relocatable values contained in this instruction.
1932 return 1; // 1 for polling page.
1933 }
1934
1935 const Pipeline * MachEpilogNode::pipeline() const {
1936 return MachNode::pipeline_class();
1937 }
1938
1939 //=============================================================================
1940
1941 static enum RC rc_class(OptoReg::Name reg) {
1942
1943 if (reg == OptoReg::Bad) {
1944 return rc_bad;
1945 }
1946
1947 // we have 32 int registers * 2 halves
1948 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1949
2208 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2209 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2210 int reg = ra_->get_encode(this);
2211
2212 // This add will handle any 24-bit signed offset. 24 bits allows an
2213 // 8 megabyte stack frame.
2214 __ add(as_Register(reg), sp, offset);
2215 }
2216
2217 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2218 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2219 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2220
2221 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2222 return NativeInstruction::instruction_size;
2223 } else {
2224 return 2 * NativeInstruction::instruction_size;
2225 }
2226 }
2227
2228 //=============================================================================
2229
2230 #ifndef PRODUCT
2231 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2232 {
2233 st->print_cr("# MachUEPNode");
2234 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2235 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2236 st->print_cr("\tcmpw rscratch1, r10");
2237 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2238 }
2239 #endif
2240
2241 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2242 {
2243 __ ic_check(InteriorEntryAlignment);
2244 }
2245
2246 uint MachUEPNode::size(PhaseRegAlloc* ra_) const
2247 {
2248 return MachNode::size(ra_);
2249 }
2250
2251 // REQUIRED EMIT CODE
2252
2253 //=============================================================================
2254
2255 // Emit deopt handler code.
2256 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2257 {
2258 // Note that the code buffer's insts_mark is always relative to insts.
2259 // That's why we must use the macroassembler to generate a handler.
2260 address base = __ start_a_stub(size_deopt_handler());
2261 if (base == nullptr) {
2262 ciEnv::current()->record_failure("CodeCache is full");
2263 return 0; // CodeBuffer::expand failed
2264 }
2265
2266 int offset = __ offset();
2267 Label start;
2268 __ bind(start);
2269 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2270
3674 %}
3675
3676 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3677 int method_index = resolved_method_index(masm);
3678 address call = __ ic_call((address)$meth$$method, method_index);
3679 if (call == nullptr) {
3680 ciEnv::current()->record_failure("CodeCache is full");
3681 return;
3682 }
3683 __ post_call_nop();
3684 if (Compile::current()->max_vector_size() > 0) {
3685 __ reinitialize_ptrue();
3686 }
3687 %}
3688
3689 enc_class aarch64_enc_call_epilog() %{
3690 if (VerifyStackAtCalls) {
3691 // Check that stack depth is unchanged: find majik cookie on stack
3692 __ call_Unimplemented();
3693 }
3694 %}
3695
3696 enc_class aarch64_enc_java_to_runtime(method meth) %{
3697 // some calls to generated routines (arraycopy code) are scheduled
3698 // by C2 as runtime calls. if so we can call them using a br (they
3699 // will be in a reachable segment) otherwise we have to use a blr
3700 // which loads the absolute address into a register.
3701 address entry = (address)$meth$$method;
3702 CodeBlob *cb = CodeCache::find_blob(entry);
3703 if (cb) {
3704 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3705 if (call == nullptr) {
3706 ciEnv::current()->record_failure("CodeCache is full");
3707 return;
3708 }
3709 __ post_call_nop();
3710 } else {
3711 Label retaddr;
3712 // Make the anchor frame walkable
3713 __ adr(rscratch2, retaddr);
3963 operand immI_gt_1()
3964 %{
3965 predicate(n->get_int() > 1);
3966 match(ConI);
3967
3968 op_cost(0);
3969 format %{ %}
3970 interface(CONST_INTER);
3971 %}
3972
3973 operand immI_le_4()
3974 %{
3975 predicate(n->get_int() <= 4);
3976 match(ConI);
3977
3978 op_cost(0);
3979 format %{ %}
3980 interface(CONST_INTER);
3981 %}
3982
3983 operand immI_16()
3984 %{
3985 predicate(n->get_int() == 16);
3986 match(ConI);
3987
3988 op_cost(0);
3989 format %{ %}
3990 interface(CONST_INTER);
3991 %}
3992
3993 operand immI_24()
3994 %{
3995 predicate(n->get_int() == 24);
3996 match(ConI);
3997
3998 op_cost(0);
3999 format %{ %}
4000 interface(CONST_INTER);
4001 %}
4002
8099 %}
8100
8101 // ============================================================================
8102 // Cast/Convert Instructions
8103
8104 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8105 match(Set dst (CastX2P src));
8106
8107 ins_cost(INSN_COST);
8108 format %{ "mov $dst, $src\t# long -> ptr" %}
8109
8110 ins_encode %{
8111 if ($dst$$reg != $src$$reg) {
8112 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8113 }
8114 %}
8115
8116 ins_pipe(ialu_reg);
8117 %}
8118
8119 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8120 match(Set dst (CastP2X src));
8121
8122 ins_cost(INSN_COST);
8123 format %{ "mov $dst, $src\t# ptr -> long" %}
8124
8125 ins_encode %{
8126 if ($dst$$reg != $src$$reg) {
8127 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8128 }
8129 %}
8130
8131 ins_pipe(ialu_reg);
8132 %}
8133
8134 // Convert oop into int for vectors alignment masking
8135 instruct convP2I(iRegINoSp dst, iRegP src) %{
8136 match(Set dst (ConvL2I (CastP2X src)));
8137
8138 ins_cost(INSN_COST);
14115
14116 match(Set dst (MoveL2D src));
14117
14118 effect(DEF dst, USE src);
14119
14120 ins_cost(INSN_COST);
14121
14122 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14123
14124 ins_encode %{
14125 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14126 %}
14127
14128 ins_pipe(fp_l2d);
14129
14130 %}
14131
14132 // ============================================================================
14133 // clearing of an array
14134
14135 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, Universe dummy, rFlagsReg cr)
14136 %{
14137 match(Set dummy (ClearArray cnt base));
14138 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14139
14140 ins_cost(4 * INSN_COST);
14141 format %{ "ClearArray $cnt, $base" %}
14142
14143 ins_encode %{
14144 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14145 if (tpc == nullptr) {
14146 ciEnv::current()->record_failure("CodeCache is full");
14147 return;
14148 }
14149 %}
14150
14151 ins_pipe(pipe_class_memory);
14152 %}
14153
14154 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, Universe dummy, rFlagsReg cr)
14155 %{
14156 predicate((uint64_t)n->in(2)->get_long()
14157 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord));
14158 match(Set dummy (ClearArray cnt base));
14159 effect(TEMP temp, USE_KILL base, KILL cr);
14160
14161 ins_cost(4 * INSN_COST);
14162 format %{ "ClearArray $cnt, $base" %}
14163
14164 ins_encode %{
14165 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14166 if (tpc == nullptr) {
14167 ciEnv::current()->record_failure("CodeCache is full");
14168 return;
14169 }
14170 %}
14171
14172 ins_pipe(pipe_class_memory);
14173 %}
14174
14175 // ============================================================================
14176 // Overflow Math Instructions
14177
14178 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15454 %}
15455
15456 // Call Runtime Instruction without safepoint and with vector arguments
15457 instruct CallLeafDirectVector(method meth)
15458 %{
15459 match(CallLeafVector);
15460
15461 effect(USE meth);
15462
15463 ins_cost(CALL_COST);
15464
15465 format %{ "CALL, runtime leaf vector $meth" %}
15466
15467 ins_encode(aarch64_enc_java_to_runtime(meth));
15468
15469 ins_pipe(pipe_class_call);
15470 %}
15471
15472 // Call Runtime Instruction
15473
15474 instruct CallLeafNoFPDirect(method meth)
15475 %{
15476 match(CallLeafNoFP);
15477
15478 effect(USE meth);
15479
15480 ins_cost(CALL_COST);
15481
15482 format %{ "CALL, runtime leaf nofp $meth" %}
15483
15484 ins_encode( aarch64_enc_java_to_runtime(meth) );
15485
15486 ins_pipe(pipe_class_call);
15487 %}
15488
15489 // Tail Call; Jump from runtime stub to Java code.
15490 // Also known as an 'interprocedural jump'.
15491 // Target of jump will eventually return to caller.
15492 // TailJump below removes the return address.
15493 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15494 // emitted just above the TailCall which has reset rfp to the caller state.
15495 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
|
1671
1672 int MachCallDynamicJavaNode::ret_addr_offset()
1673 {
1674 return 16; // movz, movk, movk, bl
1675 }
1676
1677 int MachCallRuntimeNode::ret_addr_offset() {
1678 // for generated stubs the call will be
1679 // bl(addr)
1680 // or with far branches
1681 // bl(trampoline_stub)
1682 // for real runtime callouts it will be six instructions
1683 // see aarch64_enc_java_to_runtime
1684 // adr(rscratch2, retaddr)
1685 // str(rscratch2, Address(rthread, JavaThread::last_Java_pc_offset()));
1686 // lea(rscratch1, RuntimeAddress(addr)
1687 // blr(rscratch1)
1688 CodeBlob *cb = CodeCache::find_blob(_entry_point);
1689 if (cb) {
1690 return 1 * NativeInstruction::instruction_size;
1691 } else if (_entry_point == nullptr) {
1692 // See CallLeafNoFPIndirect
1693 return 1 * NativeInstruction::instruction_size;
1694 } else {
1695 return 6 * NativeInstruction::instruction_size;
1696 }
1697 }
1698
1699 //=============================================================================
1700
1701 #ifndef PRODUCT
1702 void MachBreakpointNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1703 st->print("BREAKPOINT");
1704 }
1705 #endif
1706
1707 void MachBreakpointNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1708 __ brk(0);
1709 }
1710
1711 uint MachBreakpointNode::size(PhaseRegAlloc *ra_) const {
1712 return MachNode::size(ra_);
1713 }
1782 if (C->stub_function() == nullptr) {
1783 st->print("\n\t");
1784 st->print("ldr rscratch1, [guard]\n\t");
1785 st->print("dmb ishld\n\t");
1786 st->print("ldr rscratch2, [rthread, #thread_disarmed_guard_value_offset]\n\t");
1787 st->print("cmp rscratch1, rscratch2\n\t");
1788 st->print("b.eq skip");
1789 st->print("\n\t");
1790 st->print("blr #nmethod_entry_barrier_stub\n\t");
1791 st->print("b skip\n\t");
1792 st->print("guard: int\n\t");
1793 st->print("\n\t");
1794 st->print("skip:\n\t");
1795 }
1796 }
1797 #endif
1798
1799 void MachPrologNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1800 Compile* C = ra_->C;
1801
1802
1803 __ verified_entry(C, 0);
1804
1805 if (C->stub_function() == nullptr) {
1806 __ entry_barrier();
1807 }
1808
1809 if (!Compile::current()->output()->in_scratch_emit_size()) {
1810 __ bind(*_verified_entry);
1811 }
1812
1813 if (VerifyStackAtCalls) {
1814 Unimplemented();
1815 }
1816
1817 C->output()->set_frame_complete(__ offset());
1818
1819 if (C->has_mach_constant_base_node()) {
1820 // NOTE: We set the table base offset here because users might be
1821 // emitted before MachConstantBaseNode.
1822 ConstantTable& constant_table = C->output()->constant_table();
1823 constant_table.set_table_base_offset(constant_table.calculate_table_base_offset());
1824 }
1825 }
1826
1827 int MachPrologNode::reloc() const
1828 {
1829 return 0;
1830 }
1831
1832 //=============================================================================
1833
1834 #ifndef PRODUCT
1835 void MachEpilogNode::format(PhaseRegAlloc *ra_, outputStream *st) const {
1836 Compile* C = ra_->C;
1837 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1838
1839 st->print("# pop frame %d\n\t",framesize);
1840
1841 if (framesize == 0) {
1842 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1843 } else if (framesize < ((1 << 9) + 2 * wordSize)) {
1844 st->print("ldp lr, rfp, [sp,#%d]\n\t", framesize - 2 * wordSize);
1845 st->print("add sp, sp, #%d\n\t", framesize);
1846 } else {
1849 st->print("ldp lr, rfp, [sp],#%d\n\t", (2 * wordSize));
1850 }
1851 if (VM_Version::use_rop_protection()) {
1852 st->print("autiaz\n\t");
1853 st->print("ldr zr, [lr]\n\t");
1854 }
1855
1856 if (do_polling() && C->is_method_compilation()) {
1857 st->print("# test polling word\n\t");
1858 st->print("ldr rscratch1, [rthread],#%d\n\t", in_bytes(JavaThread::polling_word_offset()));
1859 st->print("cmp sp, rscratch1\n\t");
1860 st->print("bhi #slow_path");
1861 }
1862 }
1863 #endif
1864
1865 void MachEpilogNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
1866 Compile* C = ra_->C;
1867 int framesize = C->output()->frame_slots() << LogBytesPerInt;
1868
1869 __ remove_frame(framesize, C->needs_stack_repair());
1870
1871 if (StackReservedPages > 0 && C->has_reserved_stack_access()) {
1872 __ reserved_stack_check();
1873 }
1874
1875 if (do_polling() && C->is_method_compilation()) {
1876 Label dummy_label;
1877 Label* code_stub = &dummy_label;
1878 if (!C->output()->in_scratch_emit_size()) {
1879 C2SafepointPollStub* stub = new (C->comp_arena()) C2SafepointPollStub(__ offset());
1880 C->output()->add_stub(stub);
1881 code_stub = &stub->entry();
1882 }
1883 __ relocate(relocInfo::poll_return_type);
1884 __ safepoint_poll(*code_stub, true /* at_return */, true /* in_nmethod */);
1885 }
1886 }
1887
1888 int MachEpilogNode::reloc() const {
1889 // Return number of relocatable values contained in this instruction.
1890 return 1; // 1 for polling page.
1891 }
1892
1893 const Pipeline * MachEpilogNode::pipeline() const {
1894 return MachNode::pipeline_class();
1895 }
1896
1897 //=============================================================================
1898
1899 static enum RC rc_class(OptoReg::Name reg) {
1900
1901 if (reg == OptoReg::Bad) {
1902 return rc_bad;
1903 }
1904
1905 // we have 32 int registers * 2 halves
1906 int slots_of_int_registers = Register::number_of_registers * Register::max_slots_per_register;
1907
2166 void BoxLockNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc *ra_) const {
2167 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2168 int reg = ra_->get_encode(this);
2169
2170 // This add will handle any 24-bit signed offset. 24 bits allows an
2171 // 8 megabyte stack frame.
2172 __ add(as_Register(reg), sp, offset);
2173 }
2174
2175 uint BoxLockNode::size(PhaseRegAlloc *ra_) const {
2176 // BoxLockNode is not a MachNode, so we can't just call MachNode::size(ra_).
2177 int offset = ra_->reg2offset(in_RegMask(0).find_first_elem());
2178
2179 if (Assembler::operand_valid_for_add_sub_immediate(offset)) {
2180 return NativeInstruction::instruction_size;
2181 } else {
2182 return 2 * NativeInstruction::instruction_size;
2183 }
2184 }
2185
2186 ///=============================================================================
2187 #ifndef PRODUCT
2188 void MachVEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2189 {
2190 st->print_cr("# MachVEPNode");
2191 if (!_verified) {
2192 st->print_cr("\t load_class");
2193 } else {
2194 st->print_cr("\t unpack_inline_arg");
2195 }
2196 }
2197 #endif
2198
2199 void MachVEPNode::emit(C2_MacroAssembler *masm, PhaseRegAlloc* ra_) const
2200 {
2201 if (!_verified) {
2202 __ ic_check(1);
2203 } else {
2204 if (ra_->C->stub_function() == nullptr) {
2205 // Emit the entry barrier in a temporary frame before unpacking because
2206 // it can deopt, which would require packing the scalarized args again.
2207 __ verified_entry(ra_->C, 0);
2208 __ entry_barrier();
2209 int framesize = ra_->C->output()->frame_slots() << LogBytesPerInt;
2210 __ remove_frame(framesize, false);
2211 }
2212 // Unpack inline type args passed as oop and then jump to
2213 // the verified entry point (skipping the unverified entry).
2214 int sp_inc = __ unpack_inline_args(ra_->C, _receiver_only);
2215 // Emit code for verified entry and save increment for stack repair on return
2216 __ verified_entry(ra_->C, sp_inc);
2217 if (Compile::current()->output()->in_scratch_emit_size()) {
2218 Label dummy_verified_entry;
2219 __ b(dummy_verified_entry);
2220 } else {
2221 __ b(*_verified_entry);
2222 }
2223 }
2224 }
2225
2226 //=============================================================================
2227 #ifndef PRODUCT
2228 void MachUEPNode::format(PhaseRegAlloc* ra_, outputStream* st) const
2229 {
2230 st->print_cr("# MachUEPNode");
2231 st->print_cr("\tldrw rscratch1, [j_rarg0 + oopDesc::klass_offset_in_bytes()]\t# compressed klass");
2232 st->print_cr("\tldrw r10, [rscratch2 + CompiledICData::speculated_klass_offset()]\t# compressed klass");
2233 st->print_cr("\tcmpw rscratch1, r10");
2234 st->print_cr("\tbne, SharedRuntime::_ic_miss_stub");
2235 }
2236 #endif
2237
2238 void MachUEPNode::emit(C2_MacroAssembler* masm, PhaseRegAlloc* ra_) const
2239 {
2240 __ ic_check(InteriorEntryAlignment);
2241 }
2242
2243 // REQUIRED EMIT CODE
2244
2245 //=============================================================================
2246
2247 // Emit deopt handler code.
2248 int HandlerImpl::emit_deopt_handler(C2_MacroAssembler* masm)
2249 {
2250 // Note that the code buffer's insts_mark is always relative to insts.
2251 // That's why we must use the macroassembler to generate a handler.
2252 address base = __ start_a_stub(size_deopt_handler());
2253 if (base == nullptr) {
2254 ciEnv::current()->record_failure("CodeCache is full");
2255 return 0; // CodeBuffer::expand failed
2256 }
2257
2258 int offset = __ offset();
2259 Label start;
2260 __ bind(start);
2261 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
2262
3666 %}
3667
3668 enc_class aarch64_enc_java_dynamic_call(method meth) %{
3669 int method_index = resolved_method_index(masm);
3670 address call = __ ic_call((address)$meth$$method, method_index);
3671 if (call == nullptr) {
3672 ciEnv::current()->record_failure("CodeCache is full");
3673 return;
3674 }
3675 __ post_call_nop();
3676 if (Compile::current()->max_vector_size() > 0) {
3677 __ reinitialize_ptrue();
3678 }
3679 %}
3680
3681 enc_class aarch64_enc_call_epilog() %{
3682 if (VerifyStackAtCalls) {
3683 // Check that stack depth is unchanged: find majik cookie on stack
3684 __ call_Unimplemented();
3685 }
3686 if (tf()->returns_inline_type_as_fields() && !_method->is_method_handle_intrinsic() && _method->return_type()->is_loaded()) {
3687 // The last return value is not set by the callee but used to pass the null marker to compiled code.
3688 // Search for the corresponding projection, get the register and emit code that initializes it.
3689 uint con = (tf()->range_cc()->cnt() - 1);
3690 for (DUIterator_Fast imax, i = fast_outs(imax); i < imax; i++) {
3691 ProjNode* proj = fast_out(i)->as_Proj();
3692 if (proj->_con == con) {
3693 // Set null marker if r0 is non-null (a non-null value is returned buffered or scalarized)
3694 OptoReg::Name optoReg = ra_->get_reg_first(proj);
3695 VMReg reg = OptoReg::as_VMReg(optoReg, ra_->_framesize, OptoReg::reg2stack(ra_->_matcher._new_SP));
3696 Register toReg = reg->is_reg() ? reg->as_Register() : rscratch1;
3697 __ cmp(r0, zr);
3698 __ cset(toReg, Assembler::NE);
3699 if (reg->is_stack()) {
3700 int st_off = reg->reg2stack() * VMRegImpl::stack_slot_size;
3701 __ str(toReg, Address(sp, st_off));
3702 }
3703 break;
3704 }
3705 }
3706 if (return_value_is_used()) {
3707 // An inline type is returned as fields in multiple registers.
3708 // R0 either contains an oop if the inline type is buffered or a pointer
3709 // to the corresponding InlineKlass with the lowest bit set to 1. Zero r0
3710 // if the lowest bit is set to allow C2 to use the oop after null checking.
3711 // r0 &= (r0 & 1) - 1
3712 __ andr(rscratch1, r0, 0x1);
3713 __ sub(rscratch1, rscratch1, 0x1);
3714 __ andr(r0, r0, rscratch1);
3715 }
3716 }
3717 %}
3718
3719 enc_class aarch64_enc_java_to_runtime(method meth) %{
3720 // some calls to generated routines (arraycopy code) are scheduled
3721 // by C2 as runtime calls. if so we can call them using a br (they
3722 // will be in a reachable segment) otherwise we have to use a blr
3723 // which loads the absolute address into a register.
3724 address entry = (address)$meth$$method;
3725 CodeBlob *cb = CodeCache::find_blob(entry);
3726 if (cb) {
3727 address call = __ trampoline_call(Address(entry, relocInfo::runtime_call_type));
3728 if (call == nullptr) {
3729 ciEnv::current()->record_failure("CodeCache is full");
3730 return;
3731 }
3732 __ post_call_nop();
3733 } else {
3734 Label retaddr;
3735 // Make the anchor frame walkable
3736 __ adr(rscratch2, retaddr);
3986 operand immI_gt_1()
3987 %{
3988 predicate(n->get_int() > 1);
3989 match(ConI);
3990
3991 op_cost(0);
3992 format %{ %}
3993 interface(CONST_INTER);
3994 %}
3995
3996 operand immI_le_4()
3997 %{
3998 predicate(n->get_int() <= 4);
3999 match(ConI);
4000
4001 op_cost(0);
4002 format %{ %}
4003 interface(CONST_INTER);
4004 %}
4005
4006 operand immI_4()
4007 %{
4008 predicate(n->get_int() == 4);
4009 match(ConI);
4010
4011 op_cost(0);
4012 format %{ %}
4013 interface(CONST_INTER);
4014 %}
4015
4016 operand immI_16()
4017 %{
4018 predicate(n->get_int() == 16);
4019 match(ConI);
4020
4021 op_cost(0);
4022 format %{ %}
4023 interface(CONST_INTER);
4024 %}
4025
4026 operand immI_24()
4027 %{
4028 predicate(n->get_int() == 24);
4029 match(ConI);
4030
4031 op_cost(0);
4032 format %{ %}
4033 interface(CONST_INTER);
4034 %}
4035
8132 %}
8133
8134 // ============================================================================
8135 // Cast/Convert Instructions
8136
8137 instruct castX2P(iRegPNoSp dst, iRegL src) %{
8138 match(Set dst (CastX2P src));
8139
8140 ins_cost(INSN_COST);
8141 format %{ "mov $dst, $src\t# long -> ptr" %}
8142
8143 ins_encode %{
8144 if ($dst$$reg != $src$$reg) {
8145 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8146 }
8147 %}
8148
8149 ins_pipe(ialu_reg);
8150 %}
8151
8152 instruct castI2N(iRegNNoSp dst, iRegI src) %{
8153 match(Set dst (CastI2N src));
8154
8155 ins_cost(INSN_COST);
8156 format %{ "mov $dst, $src\t# int -> narrow ptr" %}
8157
8158 ins_encode %{
8159 if ($dst$$reg != $src$$reg) {
8160 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8161 }
8162 %}
8163
8164 ins_pipe(ialu_reg);
8165 %}
8166
8167 instruct castN2X(iRegLNoSp dst, iRegN src) %{
8168 match(Set dst (CastP2X src));
8169
8170 ins_cost(INSN_COST);
8171 format %{ "mov $dst, $src\t# ptr -> long" %}
8172
8173 ins_encode %{
8174 if ($dst$$reg != $src$$reg) {
8175 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8176 }
8177 %}
8178
8179 ins_pipe(ialu_reg);
8180 %}
8181
8182 instruct castP2X(iRegLNoSp dst, iRegP src) %{
8183 match(Set dst (CastP2X src));
8184
8185 ins_cost(INSN_COST);
8186 format %{ "mov $dst, $src\t# ptr -> long" %}
8187
8188 ins_encode %{
8189 if ($dst$$reg != $src$$reg) {
8190 __ mov(as_Register($dst$$reg), as_Register($src$$reg));
8191 }
8192 %}
8193
8194 ins_pipe(ialu_reg);
8195 %}
8196
8197 // Convert oop into int for vectors alignment masking
8198 instruct convP2I(iRegINoSp dst, iRegP src) %{
8199 match(Set dst (ConvL2I (CastP2X src)));
8200
8201 ins_cost(INSN_COST);
14178
14179 match(Set dst (MoveL2D src));
14180
14181 effect(DEF dst, USE src);
14182
14183 ins_cost(INSN_COST);
14184
14185 format %{ "fmovd $dst, $src\t# MoveL2D_reg_reg" %}
14186
14187 ins_encode %{
14188 __ fmovd(as_FloatRegister($dst$$reg), $src$$Register);
14189 %}
14190
14191 ins_pipe(fp_l2d);
14192
14193 %}
14194
14195 // ============================================================================
14196 // clearing of an array
14197
14198 instruct clearArray_reg_reg_immL0(iRegL_R11 cnt, iRegP_R10 base, immL0 zero, Universe dummy, rFlagsReg cr)
14199 %{
14200 match(Set dummy (ClearArray (Binary cnt base) zero));
14201 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14202
14203 ins_cost(4 * INSN_COST);
14204 format %{ "ClearArray $cnt, $base" %}
14205
14206 ins_encode %{
14207 address tpc = __ zero_words($base$$Register, $cnt$$Register);
14208 if (tpc == nullptr) {
14209 ciEnv::current()->record_failure("CodeCache is full");
14210 return;
14211 }
14212 %}
14213
14214 ins_pipe(pipe_class_memory);
14215 %}
14216
14217 instruct clearArray_reg_reg(iRegL_R11 cnt, iRegP_R10 base, iRegL val, Universe dummy, rFlagsReg cr)
14218 %{
14219 predicate(((ClearArrayNode*)n)->word_copy_only());
14220 match(Set dummy (ClearArray (Binary cnt base) val));
14221 effect(USE_KILL cnt, USE_KILL base, KILL cr);
14222
14223 ins_cost(4 * INSN_COST);
14224 format %{ "ClearArray $cnt, $base, $val" %}
14225
14226 ins_encode %{
14227 __ fill_words($base$$Register, $cnt$$Register, $val$$Register);
14228 %}
14229
14230 ins_pipe(pipe_class_memory);
14231 %}
14232
14233 instruct clearArray_imm_reg(immL cnt, iRegP_R10 base, iRegL_R11 temp, immL0 zero, Universe dummy, rFlagsReg cr)
14234 %{
14235 predicate((uint64_t)n->in(2)->in(1)->get_long()
14236 < (uint64_t)(BlockZeroingLowLimit >> LogBytesPerWord)
14237 && !((ClearArrayNode*)n)->word_copy_only());
14238 match(Set dummy (ClearArray (Binary cnt base) zero));
14239 effect(TEMP temp, USE_KILL base, KILL cr);
14240
14241 ins_cost(4 * INSN_COST);
14242 format %{ "ClearArray $cnt, $base" %}
14243
14244 ins_encode %{
14245 address tpc = __ zero_words($base$$Register, (uint64_t)$cnt$$constant);
14246 if (tpc == nullptr) {
14247 ciEnv::current()->record_failure("CodeCache is full");
14248 return;
14249 }
14250 %}
14251
14252 ins_pipe(pipe_class_memory);
14253 %}
14254
14255 // ============================================================================
14256 // Overflow Math Instructions
14257
14258 instruct overflowAddI_reg_reg(rFlagsReg cr, iRegIorL2I op1, iRegIorL2I op2)
15534 %}
15535
15536 // Call Runtime Instruction without safepoint and with vector arguments
15537 instruct CallLeafDirectVector(method meth)
15538 %{
15539 match(CallLeafVector);
15540
15541 effect(USE meth);
15542
15543 ins_cost(CALL_COST);
15544
15545 format %{ "CALL, runtime leaf vector $meth" %}
15546
15547 ins_encode(aarch64_enc_java_to_runtime(meth));
15548
15549 ins_pipe(pipe_class_call);
15550 %}
15551
15552 // Call Runtime Instruction
15553
15554 // entry point is null, target holds the address to call
15555 instruct CallLeafNoFPIndirect(iRegP target)
15556 %{
15557 predicate(n->as_Call()->entry_point() == nullptr);
15558
15559 match(CallLeafNoFP target);
15560
15561 ins_cost(CALL_COST);
15562
15563 format %{ "CALL, runtime leaf nofp indirect $target" %}
15564
15565 ins_encode %{
15566 __ blr($target$$Register);
15567 %}
15568
15569 ins_pipe(pipe_class_call);
15570 %}
15571
15572 instruct CallLeafNoFPDirect(method meth)
15573 %{
15574 predicate(n->as_Call()->entry_point() != nullptr);
15575
15576 match(CallLeafNoFP);
15577
15578 effect(USE meth);
15579
15580 ins_cost(CALL_COST);
15581
15582 format %{ "CALL, runtime leaf nofp $meth" %}
15583
15584 ins_encode( aarch64_enc_java_to_runtime(meth) );
15585
15586 ins_pipe(pipe_class_call);
15587 %}
15588
15589 // Tail Call; Jump from runtime stub to Java code.
15590 // Also known as an 'interprocedural jump'.
15591 // Target of jump will eventually return to caller.
15592 // TailJump below removes the return address.
15593 // Don't use rfp for 'jump_target' because a MachEpilogNode has already been
15594 // emitted just above the TailCall which has reset rfp to the caller state.
15595 instruct TailCalljmpInd(iRegPNoSpNoRfp jump_target, inline_cache_RegP method_ptr)
|