< prev index next >

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

Print this page

  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include <sys/types.h>
  27 
  28 #include "precompiled.hpp"
  29 #include "asm/assembler.hpp"
  30 #include "asm/assembler.inline.hpp"
  31 #include "ci/ciEnv.hpp"

  32 #include "compiler/compileTask.hpp"
  33 #include "compiler/disassembler.hpp"
  34 #include "compiler/oopMap.hpp"
  35 #include "gc/shared/barrierSet.hpp"
  36 #include "gc/shared/barrierSetAssembler.hpp"
  37 #include "gc/shared/cardTableBarrierSet.hpp"
  38 #include "gc/shared/cardTable.hpp"
  39 #include "gc/shared/collectedHeap.hpp"
  40 #include "gc/shared/tlab_globals.hpp"
  41 #include "interpreter/bytecodeHistogram.hpp"
  42 #include "interpreter/interpreter.hpp"
  43 #include "jvm.h"
  44 #include "memory/resourceArea.hpp"
  45 #include "memory/universe.hpp"
  46 #include "nativeInst_aarch64.hpp"
  47 #include "oops/accessDecorators.hpp"
  48 #include "oops/compressedKlass.inline.hpp"
  49 #include "oops/compressedOops.inline.hpp"
  50 #include "oops/klass.inline.hpp"

  51 #include "runtime/continuation.hpp"
  52 #include "runtime/icache.hpp"
  53 #include "runtime/interfaceSupport.inline.hpp"
  54 #include "runtime/javaThread.hpp"
  55 #include "runtime/jniHandles.inline.hpp"
  56 #include "runtime/sharedRuntime.hpp"

  57 #include "runtime/stubRoutines.hpp"
  58 #include "utilities/powerOfTwo.hpp"

  59 #ifdef COMPILER1
  60 #include "c1/c1_LIRAssembler.hpp"
  61 #endif
  62 #ifdef COMPILER2
  63 #include "oops/oop.hpp"
  64 #include "opto/compile.hpp"
  65 #include "opto/node.hpp"
  66 #include "opto/output.hpp"
  67 #endif
  68 
  69 #ifdef PRODUCT
  70 #define BLOCK_COMMENT(str) /* nothing */
  71 #else
  72 #define BLOCK_COMMENT(str) block_comment(str)
  73 #endif
  74 #define STOP(str) stop(str);
  75 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  76 
  77 #ifdef ASSERT
  78 extern "C" void disnm(intptr_t p);

1104 }
1105 
1106 void MacroAssembler::post_call_nop() {
1107   if (!Continuations::enabled()) {
1108     return;
1109   }
1110   InstructionMark im(this);
1111   relocate(post_call_nop_Relocation::spec());
1112   InlineSkippedInstructionsCounter skipCounter(this);
1113   nop();
1114   movk(zr, 0);
1115   movk(zr, 0);
1116 }
1117 
1118 // these are no-ops overridden by InterpreterMacroAssembler
1119 
1120 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { }
1121 
1122 void MacroAssembler::check_and_handle_popframe(Register java_thread) { }
1123 



































1124 // Look up the method for a megamorphic invokeinterface call.
1125 // The target method is determined by <intf_klass, itable_index>.
1126 // The receiver klass is in recv_klass.
1127 // On success, the result will be in method_result, and execution falls through.
1128 // On failure, execution transfers to the given label.
1129 void MacroAssembler::lookup_interface_method(Register recv_klass,
1130                                              Register intf_klass,
1131                                              RegisterOrConstant itable_index,
1132                                              Register method_result,
1133                                              Register scan_temp,
1134                                              Label& L_no_such_interface,
1135                          bool return_method) {
1136   assert_different_registers(recv_klass, intf_klass, scan_temp);
1137   assert_different_registers(method_result, intf_klass, scan_temp);
1138   assert(recv_klass != method_result || !return_method,
1139      "recv_klass can be destroyed when method isn't needed");
1140   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
1141          "caller must use same register for non-constant itable index as for method");
1142 
1143   // Compute start of first itableOffsetEntry (which is at the end of the vtable)

1557   ldrb(scratch, Address(klass, InstanceKlass::init_state_offset()));
1558   subs(zr, scratch, InstanceKlass::fully_initialized);
1559   br(Assembler::EQ, *L_fast_path);
1560 
1561   // Fast path check: current thread is initializer thread
1562   ldr(scratch, Address(klass, InstanceKlass::init_thread_offset()));
1563   cmp(rthread, scratch);
1564 
1565   if (L_slow_path == &L_fallthrough) {
1566     br(Assembler::EQ, *L_fast_path);
1567     bind(*L_slow_path);
1568   } else if (L_fast_path == &L_fallthrough) {
1569     br(Assembler::NE, *L_slow_path);
1570     bind(*L_fast_path);
1571   } else {
1572     Unimplemented();
1573   }
1574 }
1575 
1576 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
1577   if (!VerifyOops) return;




1578 
1579   // Pass register number to verify_oop_subroutine
1580   const char* b = nullptr;
1581   {
1582     ResourceMark rm;
1583     stringStream ss;
1584     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
1585     b = code_string(ss.as_string());
1586   }
1587   BLOCK_COMMENT("verify_oop {");
1588 
1589   strip_return_address(); // This might happen within a stack frame.
1590   protect_return_address();
1591   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1592   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1593 
1594   mov(r0, reg);
1595   movptr(rscratch1, (uintptr_t)(address)b);
1596 
1597   // call indirectly to solve generation ordering problem
1598   lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1599   ldr(rscratch2, Address(rscratch2));
1600   blr(rscratch2);
1601 
1602   ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1603   ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1604   authenticate_return_address();
1605 
1606   BLOCK_COMMENT("} verify_oop");
1607 }
1608 
1609 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
1610   if (!VerifyOops) return;




1611 
1612   const char* b = nullptr;
1613   {
1614     ResourceMark rm;
1615     stringStream ss;
1616     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
1617     b = code_string(ss.as_string());
1618   }
1619   BLOCK_COMMENT("verify_oop_addr {");
1620 
1621   strip_return_address(); // This might happen within a stack frame.
1622   protect_return_address();
1623   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1624   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1625 
1626   // addr may contain sp so we will have to adjust it based on the
1627   // pushes that we just did.
1628   if (addr.uses(sp)) {
1629     lea(r0, addr);
1630     ldr(r0, Address(r0, 4 * wordSize));

1688   call_VM_leaf_base(entry_point, 1);
1689 }
1690 
1691 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1692   assert_different_registers(arg_1, c_rarg0);
1693   pass_arg0(this, arg_0);
1694   pass_arg1(this, arg_1);
1695   call_VM_leaf_base(entry_point, 2);
1696 }
1697 
1698 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
1699                                   Register arg_1, Register arg_2) {
1700   assert_different_registers(arg_1, c_rarg0);
1701   assert_different_registers(arg_2, c_rarg0, c_rarg1);
1702   pass_arg0(this, arg_0);
1703   pass_arg1(this, arg_1);
1704   pass_arg2(this, arg_2);
1705   call_VM_leaf_base(entry_point, 3);
1706 }
1707 




1708 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1709   pass_arg0(this, arg_0);
1710   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1711 }
1712 
1713 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1714 
1715   assert_different_registers(arg_0, c_rarg1);
1716   pass_arg1(this, arg_1);
1717   pass_arg0(this, arg_0);
1718   MacroAssembler::call_VM_leaf_base(entry_point, 2);
1719 }
1720 
1721 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1722   assert_different_registers(arg_0, c_rarg1, c_rarg2);
1723   assert_different_registers(arg_1, c_rarg2);
1724   pass_arg2(this, arg_2);
1725   pass_arg1(this, arg_1);
1726   pass_arg0(this, arg_0);
1727   MacroAssembler::call_VM_leaf_base(entry_point, 3);

1733   assert_different_registers(arg_2, c_rarg3);
1734   pass_arg3(this, arg_3);
1735   pass_arg2(this, arg_2);
1736   pass_arg1(this, arg_1);
1737   pass_arg0(this, arg_0);
1738   MacroAssembler::call_VM_leaf_base(entry_point, 4);
1739 }
1740 
1741 void MacroAssembler::null_check(Register reg, int offset) {
1742   if (needs_explicit_null_check(offset)) {
1743     // provoke OS null exception if reg is null by
1744     // accessing M[reg] w/o changing any registers
1745     // NOTE: this is plenty to provoke a segv
1746     ldr(zr, Address(reg));
1747   } else {
1748     // nothing to do, (later) access of M[reg + offset]
1749     // will provoke OS null exception if reg is null
1750   }
1751 }
1752 














































































































1753 // MacroAssembler protected routines needed to implement
1754 // public methods
1755 
1756 void MacroAssembler::mov(Register r, Address dest) {
1757   code_section()->relocate(pc(), dest.rspec());
1758   uint64_t imm64 = (uint64_t)dest.target();
1759   movptr(r, imm64);
1760 }
1761 
1762 // Move a constant pointer into r.  In AArch64 mode the virtual
1763 // address space is 48 bits in size, so we only need three
1764 // instructions to create a patchable instruction sequence that can
1765 // reach anywhere.
1766 void MacroAssembler::movptr(Register r, uintptr_t imm64) {
1767 #ifndef PRODUCT
1768   {
1769     char buffer[64];
1770     snprintf(buffer, sizeof(buffer), "0x%" PRIX64, (uint64_t)imm64);
1771     block_comment(buffer);
1772   }

4402   adrp(rscratch1, src2, offset);
4403   ldr(rscratch1, Address(rscratch1, offset));
4404   cmp(src1, rscratch1);
4405 }
4406 
4407 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
4408   cmp(obj1, obj2);
4409 }
4410 
4411 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4412   load_method_holder(rresult, rmethod);
4413   ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4414 }
4415 
4416 void MacroAssembler::load_method_holder(Register holder, Register method) {
4417   ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
4418   ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
4419   ldr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
4420 }
4421 








4422 void MacroAssembler::load_klass(Register dst, Register src) {
4423   if (UseCompressedClassPointers) {
4424     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4425     decode_klass_not_null(dst);
4426   } else {
4427     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4428   }
4429 }
4430 
4431 // ((OopHandle)result).resolve();
4432 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) {
4433   // OopHandle::resolve is an indirection.
4434   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp1, tmp2);
4435 }
4436 
4437 // ((WeakHandle)result).resolve();
4438 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2) {
4439   assert_different_registers(result, tmp1, tmp2);
4440   Label resolved;
4441 

4460 
4461 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
4462   if (UseCompressedClassPointers) {
4463     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4464     if (CompressedKlassPointers::base() == nullptr) {
4465       cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
4466       return;
4467     } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
4468                && CompressedKlassPointers::shift() == 0) {
4469       // Only the bottom 32 bits matter
4470       cmpw(trial_klass, tmp);
4471       return;
4472     }
4473     decode_klass_not_null(tmp);
4474   } else {
4475     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4476   }
4477   cmp(trial_klass, tmp);
4478 }
4479 





4480 void MacroAssembler::store_klass(Register dst, Register src) {
4481   // FIXME: Should this be a store release?  concurrent gcs assumes
4482   // klass length is valid if klass field is not null.
4483   if (UseCompressedClassPointers) {
4484     encode_klass_not_null(src);
4485     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4486   } else {
4487     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4488   }
4489 }
4490 
4491 void MacroAssembler::store_klass_gap(Register dst, Register src) {
4492   if (UseCompressedClassPointers) {
4493     // Store to klass gap in destination
4494     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
4495   }
4496 }
4497 
4498 // Algorithm must match CompressedOops::encode.
4499 void MacroAssembler::encode_heap_oop(Register d, Register s) {

4784   if (as_raw) {
4785     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2);
4786   } else {
4787     bs->load_at(this, decorators, type, dst, src, tmp1, tmp2);
4788   }
4789 }
4790 
4791 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
4792                                      Address dst, Register val,
4793                                      Register tmp1, Register tmp2, Register tmp3) {
4794   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4795   decorators = AccessInternal::decorator_fixup(decorators, type);
4796   bool as_raw = (decorators & AS_RAW) != 0;
4797   if (as_raw) {
4798     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
4799   } else {
4800     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
4801   }
4802 }
4803 








































4804 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
4805                                    Register tmp2, DecoratorSet decorators) {
4806   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
4807 }
4808 
4809 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
4810                                             Register tmp2, DecoratorSet decorators) {
4811   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, tmp2);
4812 }
4813 
4814 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
4815                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
4816   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
4817 }
4818 
4819 // Used for storing nulls.
4820 void MacroAssembler::store_heap_oop_null(Address dst) {
4821   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
4822 }
4823 

4860     oop_index = oop_recorder()->allocate_metadata_index(obj);
4861   } else {
4862     oop_index = oop_recorder()->find_index(obj);
4863   }
4864   RelocationHolder rspec = metadata_Relocation::spec(oop_index);
4865   mov(dst, Address((address)obj, rspec));
4866 }
4867 
4868 Address MacroAssembler::constant_oop_address(jobject obj) {
4869 #ifdef ASSERT
4870   {
4871     ThreadInVMfromUnknown tiv;
4872     assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder");
4873     assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "not an oop");
4874   }
4875 #endif
4876   int oop_index = oop_recorder()->find_index(obj);
4877   return Address((address)obj, oop_Relocation::spec(oop_index));
4878 }
4879 
































































































4880 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4881 void MacroAssembler::tlab_allocate(Register obj,
4882                                    Register var_size_in_bytes,
4883                                    int con_size_in_bytes,
4884                                    Register t1,
4885                                    Register t2,
4886                                    Label& slow_case) {
4887   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4888   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
4889 }
4890 
4891 void MacroAssembler::verify_tlab() {
4892 #ifdef ASSERT
4893   if (UseTLAB && VerifyOops) {
4894     Label next, ok;
4895 
4896     stp(rscratch2, rscratch1, Address(pre(sp, -16)));
4897 
4898     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4899     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4900     cmp(rscratch2, rscratch1);
4901     br(Assembler::HS, next);
4902     STOP("assert(top >= start)");
4903     should_not_reach_here();
4904 
4905     bind(next);
4906     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4907     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4908     cmp(rscratch2, rscratch1);
4909     br(Assembler::HS, ok);
4910     STOP("assert(top <= end)");
4911     should_not_reach_here();
4912 
4913     bind(ok);
4914     ldp(rscratch2, rscratch1, Address(post(sp, 16)));
4915   }
4916 #endif
4917 }
4918 













4919 // Writes to stack successive pages until offset reached to check for
4920 // stack overflow + shadow pages.  This clobbers tmp.
4921 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
4922   assert_different_registers(tmp, size, rscratch1);
4923   mov(tmp, sp);
4924   // Bang stack for total size given plus shadow page size.
4925   // Bang one page at a time because large size can bang beyond yellow and
4926   // red zones.
4927   Label loop;
4928   mov(rscratch1, (int)os::vm_page_size());
4929   bind(loop);
4930   lea(tmp, Address(tmp, -(int)os::vm_page_size()));
4931   subsw(size, size, rscratch1);
4932   str(size, Address(tmp));
4933   br(Assembler::GT, loop);
4934 
4935   // Bang down shadow pages too.
4936   // At this point, (tmp-0) is the last address touched, so don't
4937   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
4938   // was post-decremented.)  Skip this address by starting at i=1, and

5024 }
5025 
5026 void MacroAssembler::remove_frame(int framesize) {
5027   assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR");
5028   assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5029   if (framesize < ((1 << 9) + 2 * wordSize)) {
5030     ldp(rfp, lr, Address(sp, framesize - 2 * wordSize));
5031     add(sp, sp, framesize);
5032   } else {
5033     if (framesize < ((1 << 12) + 2 * wordSize))
5034       add(sp, sp, framesize - 2 * wordSize);
5035     else {
5036       mov(rscratch1, framesize - 2 * wordSize);
5037       add(sp, sp, rscratch1);
5038     }
5039     ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
5040   }
5041   authenticate_return_address();
5042 }
5043 



















































5044 
5045 // This method counts leading positive bytes (highest bit not set) in provided byte array
5046 address MacroAssembler::count_positives(Register ary1, Register len, Register result) {
5047     // Simple and most common case of aligned small array which is not at the
5048     // end of memory page is placed here. All other cases are in stub.
5049     Label LOOP, END, STUB, STUB_LONG, SET_RESULT, DONE;
5050     const uint64_t UPPER_BIT_MASK=0x8080808080808080;
5051     assert_different_registers(ary1, len, result);
5052 
5053     mov(result, len);
5054     cmpw(len, 0);
5055     br(LE, DONE);
5056     cmpw(len, 4 * wordSize);
5057     br(GE, STUB_LONG); // size > 32 then go to stub
5058 
5059     int shift = 64 - exact_log2(os::vm_page_size());
5060     lsl(rscratch1, ary1, shift);
5061     mov(rscratch2, (size_t)(4 * wordSize) << shift);
5062     adds(rscratch2, rscratch1, rscratch2);  // At end of page?
5063     br(CS, STUB); // at the end of page then go to stub

5938 // On other systems, the helper is a usual C function.
5939 //
5940 void MacroAssembler::get_thread(Register dst) {
5941   RegSet saved_regs =
5942     LINUX_ONLY(RegSet::range(r0, r1)  + lr - dst)
5943     NOT_LINUX (RegSet::range(r0, r17) + lr - dst);
5944 
5945   protect_return_address();
5946   push(saved_regs, sp);
5947 
5948   mov(lr, CAST_FROM_FN_PTR(address, JavaThread::aarch64_get_thread_helper));
5949   blr(lr);
5950   if (dst != c_rarg0) {
5951     mov(dst, c_rarg0);
5952   }
5953 
5954   pop(saved_regs, sp);
5955   authenticate_return_address();
5956 }
5957 

























































































































































































































































































































































































































































5958 void MacroAssembler::cache_wb(Address line) {
5959   assert(line.getMode() == Address::base_plus_offset, "mode should be base_plus_offset");
5960   assert(line.index() == noreg, "index should be noreg");
5961   assert(line.offset() == 0, "offset should be 0");
5962   // would like to assert this
5963   // assert(line._ext.shift == 0, "shift should be zero");
5964   if (VM_Version::supports_dcpop()) {
5965     // writeback using clear virtual address to point of persistence
5966     dc(Assembler::CVAP, line.base());
5967   } else {
5968     // no need to generate anything as Unsafe.writebackMemory should
5969     // never invoke this stub
5970   }
5971 }
5972 
5973 void MacroAssembler::cache_wbsync(bool is_pre) {
5974   // we only need a barrier post sync
5975   if (!is_pre) {
5976     membar(Assembler::AnyAny);
5977   }

6311 }
6312 
6313 // Implements lightweight-locking.
6314 // Branches to slow upon failure to lock the object, with ZF cleared.
6315 // Falls through upon success with ZF set.
6316 //
6317 //  - obj: the object to be locked
6318 //  - hdr: the header, already loaded from obj, will be destroyed
6319 //  - t1, t2: temporary registers, will be destroyed
6320 void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
6321   assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
6322   assert_different_registers(obj, hdr, t1, t2, rscratch1);
6323 
6324   // Check if we would have space on lock-stack for the object.
6325   ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
6326   cmpw(t1, (unsigned)LockStack::end_offset() - 1);
6327   br(Assembler::GT, slow);
6328 
6329   // Load (object->mark() | 1) into hdr
6330   orr(hdr, hdr, markWord::unlocked_value);





6331   // Clear lock-bits, into t2
6332   eor(t2, hdr, markWord::unlocked_value);
6333   // Try to swing header from unlocked to locked
6334   // Clobbers rscratch1 when UseLSE is false
6335   cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
6336           /*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
6337   br(Assembler::NE, slow);
6338 
6339   // After successful lock, push object on lock-stack
6340   ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
6341   str(obj, Address(rthread, t1));
6342   addw(t1, t1, oopSize);
6343   strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
6344 }
6345 
6346 // Implements lightweight-unlocking.
6347 // Branches to slow upon failure, with ZF cleared.
6348 // Falls through upon success, with ZF set.
6349 //
6350 // - obj: the object to be unlocked

  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include <sys/types.h>
  27 
  28 #include "precompiled.hpp"
  29 #include "asm/assembler.hpp"
  30 #include "asm/assembler.inline.hpp"
  31 #include "ci/ciEnv.hpp"
  32 #include "ci/ciInlineKlass.hpp"
  33 #include "compiler/compileTask.hpp"
  34 #include "compiler/disassembler.hpp"
  35 #include "compiler/oopMap.hpp"
  36 #include "gc/shared/barrierSet.hpp"
  37 #include "gc/shared/barrierSetAssembler.hpp"
  38 #include "gc/shared/cardTableBarrierSet.hpp"
  39 #include "gc/shared/cardTable.hpp"
  40 #include "gc/shared/collectedHeap.hpp"
  41 #include "gc/shared/tlab_globals.hpp"
  42 #include "interpreter/bytecodeHistogram.hpp"
  43 #include "interpreter/interpreter.hpp"
  44 #include "jvm.h"
  45 #include "memory/resourceArea.hpp"
  46 #include "memory/universe.hpp"
  47 #include "nativeInst_aarch64.hpp"
  48 #include "oops/accessDecorators.hpp"
  49 #include "oops/compressedKlass.inline.hpp"
  50 #include "oops/compressedOops.inline.hpp"
  51 #include "oops/klass.inline.hpp"
  52 #include "oops/resolvedFieldEntry.hpp"
  53 #include "runtime/continuation.hpp"
  54 #include "runtime/icache.hpp"
  55 #include "runtime/interfaceSupport.inline.hpp"
  56 #include "runtime/javaThread.hpp"
  57 #include "runtime/jniHandles.inline.hpp"
  58 #include "runtime/sharedRuntime.hpp"
  59 #include "runtime/signature_cc.hpp"
  60 #include "runtime/stubRoutines.hpp"
  61 #include "utilities/powerOfTwo.hpp"
  62 #include "vmreg_aarch64.inline.hpp"
  63 #ifdef COMPILER1
  64 #include "c1/c1_LIRAssembler.hpp"
  65 #endif
  66 #ifdef COMPILER2
  67 #include "oops/oop.hpp"
  68 #include "opto/compile.hpp"
  69 #include "opto/node.hpp"
  70 #include "opto/output.hpp"
  71 #endif
  72 
  73 #ifdef PRODUCT
  74 #define BLOCK_COMMENT(str) /* nothing */
  75 #else
  76 #define BLOCK_COMMENT(str) block_comment(str)
  77 #endif
  78 #define STOP(str) stop(str);
  79 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  80 
  81 #ifdef ASSERT
  82 extern "C" void disnm(intptr_t p);

1108 }
1109 
1110 void MacroAssembler::post_call_nop() {
1111   if (!Continuations::enabled()) {
1112     return;
1113   }
1114   InstructionMark im(this);
1115   relocate(post_call_nop_Relocation::spec());
1116   InlineSkippedInstructionsCounter skipCounter(this);
1117   nop();
1118   movk(zr, 0);
1119   movk(zr, 0);
1120 }
1121 
1122 // these are no-ops overridden by InterpreterMacroAssembler
1123 
1124 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { }
1125 
1126 void MacroAssembler::check_and_handle_popframe(Register java_thread) { }
1127 
1128 void MacroAssembler::get_default_value_oop(Register inline_klass, Register temp_reg, Register obj) {
1129 #ifdef ASSERT
1130   {
1131     Label done_check;
1132     test_klass_is_inline_type(inline_klass, temp_reg, done_check);
1133     stop("get_default_value_oop from non inline type klass");
1134     bind(done_check);
1135   }
1136 #endif
1137   Register offset = temp_reg;
1138   // Getting the offset of the pre-allocated default value
1139   ldr(offset, Address(inline_klass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())));
1140   ldr(offset, Address(offset, in_bytes(InlineKlass::default_value_offset_offset())));
1141 
1142   // Getting the mirror
1143   ldr(obj, Address(inline_klass, in_bytes(Klass::java_mirror_offset())));
1144   resolve_oop_handle(obj, inline_klass, temp_reg);
1145 
1146   // Getting the pre-allocated default value from the mirror
1147   Address field(obj, offset);
1148   load_heap_oop(obj, field, inline_klass, rscratch2);
1149 }
1150 
1151 void MacroAssembler::get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj) {
1152 #ifdef ASSERT
1153   {
1154     Label done_check;
1155     test_klass_is_empty_inline_type(inline_klass, temp_reg, done_check);
1156     stop("get_empty_value from non-empty inline klass");
1157     bind(done_check);
1158   }
1159 #endif
1160   get_default_value_oop(inline_klass, temp_reg, obj);
1161 }
1162 
1163 // Look up the method for a megamorphic invokeinterface call.
1164 // The target method is determined by <intf_klass, itable_index>.
1165 // The receiver klass is in recv_klass.
1166 // On success, the result will be in method_result, and execution falls through.
1167 // On failure, execution transfers to the given label.
1168 void MacroAssembler::lookup_interface_method(Register recv_klass,
1169                                              Register intf_klass,
1170                                              RegisterOrConstant itable_index,
1171                                              Register method_result,
1172                                              Register scan_temp,
1173                                              Label& L_no_such_interface,
1174                          bool return_method) {
1175   assert_different_registers(recv_klass, intf_klass, scan_temp);
1176   assert_different_registers(method_result, intf_klass, scan_temp);
1177   assert(recv_klass != method_result || !return_method,
1178      "recv_klass can be destroyed when method isn't needed");
1179   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
1180          "caller must use same register for non-constant itable index as for method");
1181 
1182   // Compute start of first itableOffsetEntry (which is at the end of the vtable)

1596   ldrb(scratch, Address(klass, InstanceKlass::init_state_offset()));
1597   subs(zr, scratch, InstanceKlass::fully_initialized);
1598   br(Assembler::EQ, *L_fast_path);
1599 
1600   // Fast path check: current thread is initializer thread
1601   ldr(scratch, Address(klass, InstanceKlass::init_thread_offset()));
1602   cmp(rthread, scratch);
1603 
1604   if (L_slow_path == &L_fallthrough) {
1605     br(Assembler::EQ, *L_fast_path);
1606     bind(*L_slow_path);
1607   } else if (L_fast_path == &L_fallthrough) {
1608     br(Assembler::NE, *L_slow_path);
1609     bind(*L_fast_path);
1610   } else {
1611     Unimplemented();
1612   }
1613 }
1614 
1615 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
1616   if (!VerifyOops || VerifyAdapterSharing) {
1617     // Below address of the code string confuses VerifyAdapterSharing
1618     // because it may differ between otherwise equivalent adapters.
1619     return;
1620   }
1621 
1622   // Pass register number to verify_oop_subroutine
1623   const char* b = nullptr;
1624   {
1625     ResourceMark rm;
1626     stringStream ss;
1627     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
1628     b = code_string(ss.as_string());
1629   }
1630   BLOCK_COMMENT("verify_oop {");
1631 
1632   strip_return_address(); // This might happen within a stack frame.
1633   protect_return_address();
1634   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1635   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1636 
1637   mov(r0, reg);
1638   movptr(rscratch1, (uintptr_t)(address)b);
1639 
1640   // call indirectly to solve generation ordering problem
1641   lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1642   ldr(rscratch2, Address(rscratch2));
1643   blr(rscratch2);
1644 
1645   ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1646   ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1647   authenticate_return_address();
1648 
1649   BLOCK_COMMENT("} verify_oop");
1650 }
1651 
1652 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
1653   if (!VerifyOops || VerifyAdapterSharing) {
1654     // Below address of the code string confuses VerifyAdapterSharing
1655     // because it may differ between otherwise equivalent adapters.
1656     return;
1657   }
1658 
1659   const char* b = nullptr;
1660   {
1661     ResourceMark rm;
1662     stringStream ss;
1663     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
1664     b = code_string(ss.as_string());
1665   }
1666   BLOCK_COMMENT("verify_oop_addr {");
1667 
1668   strip_return_address(); // This might happen within a stack frame.
1669   protect_return_address();
1670   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1671   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1672 
1673   // addr may contain sp so we will have to adjust it based on the
1674   // pushes that we just did.
1675   if (addr.uses(sp)) {
1676     lea(r0, addr);
1677     ldr(r0, Address(r0, 4 * wordSize));

1735   call_VM_leaf_base(entry_point, 1);
1736 }
1737 
1738 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1739   assert_different_registers(arg_1, c_rarg0);
1740   pass_arg0(this, arg_0);
1741   pass_arg1(this, arg_1);
1742   call_VM_leaf_base(entry_point, 2);
1743 }
1744 
1745 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
1746                                   Register arg_1, Register arg_2) {
1747   assert_different_registers(arg_1, c_rarg0);
1748   assert_different_registers(arg_2, c_rarg0, c_rarg1);
1749   pass_arg0(this, arg_0);
1750   pass_arg1(this, arg_1);
1751   pass_arg2(this, arg_2);
1752   call_VM_leaf_base(entry_point, 3);
1753 }
1754 
1755 void MacroAssembler::super_call_VM_leaf(address entry_point) {
1756   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1757 }
1758 
1759 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1760   pass_arg0(this, arg_0);
1761   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1762 }
1763 
1764 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1765 
1766   assert_different_registers(arg_0, c_rarg1);
1767   pass_arg1(this, arg_1);
1768   pass_arg0(this, arg_0);
1769   MacroAssembler::call_VM_leaf_base(entry_point, 2);
1770 }
1771 
1772 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1773   assert_different_registers(arg_0, c_rarg1, c_rarg2);
1774   assert_different_registers(arg_1, c_rarg2);
1775   pass_arg2(this, arg_2);
1776   pass_arg1(this, arg_1);
1777   pass_arg0(this, arg_0);
1778   MacroAssembler::call_VM_leaf_base(entry_point, 3);

1784   assert_different_registers(arg_2, c_rarg3);
1785   pass_arg3(this, arg_3);
1786   pass_arg2(this, arg_2);
1787   pass_arg1(this, arg_1);
1788   pass_arg0(this, arg_0);
1789   MacroAssembler::call_VM_leaf_base(entry_point, 4);
1790 }
1791 
1792 void MacroAssembler::null_check(Register reg, int offset) {
1793   if (needs_explicit_null_check(offset)) {
1794     // provoke OS null exception if reg is null by
1795     // accessing M[reg] w/o changing any registers
1796     // NOTE: this is plenty to provoke a segv
1797     ldr(zr, Address(reg));
1798   } else {
1799     // nothing to do, (later) access of M[reg + offset]
1800     // will provoke OS null exception if reg is null
1801   }
1802 }
1803 
1804 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
1805   assert_different_registers(markword, rscratch2);
1806   andr(markword, markword, markWord::inline_type_mask_in_place);
1807   mov(rscratch2, markWord::inline_type_pattern);
1808   cmp(markword, rscratch2);
1809   br(Assembler::EQ, is_inline_type);
1810 }
1811 
1812 void MacroAssembler::test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type) {
1813   ldrw(temp_reg, Address(klass, Klass::access_flags_offset()));
1814   andr(temp_reg, temp_reg, JVM_ACC_VALUE);
1815   cbnz(temp_reg, is_inline_type);
1816 }
1817 
1818 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type) {
1819   assert_different_registers(tmp, rscratch1);
1820   cbz(object, not_inline_type);
1821   const int is_inline_type_mask = markWord::inline_type_pattern;
1822   ldr(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
1823   mov(rscratch1, is_inline_type_mask);
1824   andr(tmp, tmp, rscratch1);
1825   cmp(tmp, rscratch1);
1826   br(Assembler::NE, not_inline_type);
1827 }
1828 
1829 void MacroAssembler::test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type) {
1830 #ifdef ASSERT
1831   {
1832     Label done_check;
1833     test_klass_is_inline_type(klass, temp_reg, done_check);
1834     stop("test_klass_is_empty_inline_type with non inline type klass");
1835     bind(done_check);
1836   }
1837 #endif
1838   ldrw(temp_reg, Address(klass, InstanceKlass::misc_flags_offset()));
1839   andr(temp_reg, temp_reg, InstanceKlassFlags::is_empty_inline_type_value());
1840   cbnz(temp_reg, is_empty_inline_type);
1841 }
1842 
1843 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
1844   assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86
1845   tbnz(flags, ResolvedFieldEntry::is_null_free_inline_type_shift, is_null_free_inline_type);
1846 }
1847 
1848 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
1849   assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86
1850   tbz(flags, ResolvedFieldEntry::is_null_free_inline_type_shift, not_null_free_inline_type);
1851 }
1852 
1853 void MacroAssembler::test_field_is_flat(Register flags, Register temp_reg, Label& is_flat) {
1854   assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86
1855   tbnz(flags, ResolvedFieldEntry::is_flat_shift, is_flat);
1856 }
1857 
1858 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label) {
1859   Label test_mark_word;
1860   // load mark word
1861   ldr(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
1862   // check displaced
1863   tst(temp_reg, markWord::unlocked_value);
1864   br(Assembler::NE, test_mark_word);
1865   // slow path use klass prototype
1866   load_prototype_header(temp_reg, oop);
1867 
1868   bind(test_mark_word);
1869   andr(temp_reg, temp_reg, test_bit);
1870   if (jmp_set) {
1871     cbnz(temp_reg, jmp_label);
1872   } else {
1873     cbz(temp_reg, jmp_label);
1874   }
1875 }
1876 
1877 void MacroAssembler::test_flat_array_oop(Register oop, Register temp_reg, Label& is_flat_array) {
1878   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flat_array);
1879 }
1880 
1881 void MacroAssembler::test_non_flat_array_oop(Register oop, Register temp_reg,
1882                                                   Label&is_non_flat_array) {
1883   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flat_array);
1884 }
1885 
1886 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array) {
1887   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
1888 }
1889 
1890 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
1891   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
1892 }
1893 
1894 void MacroAssembler::test_flat_array_layout(Register lh, Label& is_flat_array) {
1895   tst(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
1896   br(Assembler::NE, is_flat_array);
1897 }
1898 
1899 void MacroAssembler::test_non_flat_array_layout(Register lh, Label& is_non_flat_array) {
1900   tst(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
1901   br(Assembler::EQ, is_non_flat_array);
1902 }
1903 
1904 void MacroAssembler::test_null_free_array_layout(Register lh, Label& is_null_free_array) {
1905   tst(lh, Klass::_lh_null_free_array_bit_inplace);
1906   br(Assembler::NE, is_null_free_array);
1907 }
1908 
1909 void MacroAssembler::test_non_null_free_array_layout(Register lh, Label& is_non_null_free_array) {
1910   tst(lh, Klass::_lh_null_free_array_bit_inplace);
1911   br(Assembler::EQ, is_non_null_free_array);
1912 }
1913 
1914 // MacroAssembler protected routines needed to implement
1915 // public methods
1916 
1917 void MacroAssembler::mov(Register r, Address dest) {
1918   code_section()->relocate(pc(), dest.rspec());
1919   uint64_t imm64 = (uint64_t)dest.target();
1920   movptr(r, imm64);
1921 }
1922 
1923 // Move a constant pointer into r.  In AArch64 mode the virtual
1924 // address space is 48 bits in size, so we only need three
1925 // instructions to create a patchable instruction sequence that can
1926 // reach anywhere.
1927 void MacroAssembler::movptr(Register r, uintptr_t imm64) {
1928 #ifndef PRODUCT
1929   {
1930     char buffer[64];
1931     snprintf(buffer, sizeof(buffer), "0x%" PRIX64, (uint64_t)imm64);
1932     block_comment(buffer);
1933   }

4563   adrp(rscratch1, src2, offset);
4564   ldr(rscratch1, Address(rscratch1, offset));
4565   cmp(src1, rscratch1);
4566 }
4567 
4568 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
4569   cmp(obj1, obj2);
4570 }
4571 
4572 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4573   load_method_holder(rresult, rmethod);
4574   ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4575 }
4576 
4577 void MacroAssembler::load_method_holder(Register holder, Register method) {
4578   ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
4579   ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
4580   ldr(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
4581 }
4582 
4583 void MacroAssembler::load_metadata(Register dst, Register src) {
4584   if (UseCompressedClassPointers) {
4585     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4586   } else {
4587     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4588   }
4589 }
4590 
4591 void MacroAssembler::load_klass(Register dst, Register src) {
4592   if (UseCompressedClassPointers) {
4593     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4594     decode_klass_not_null(dst);
4595   } else {
4596     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4597   }
4598 }
4599 
4600 // ((OopHandle)result).resolve();
4601 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) {
4602   // OopHandle::resolve is an indirection.
4603   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp1, tmp2);
4604 }
4605 
4606 // ((WeakHandle)result).resolve();
4607 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2) {
4608   assert_different_registers(result, tmp1, tmp2);
4609   Label resolved;
4610 

4629 
4630 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
4631   if (UseCompressedClassPointers) {
4632     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4633     if (CompressedKlassPointers::base() == nullptr) {
4634       cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
4635       return;
4636     } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
4637                && CompressedKlassPointers::shift() == 0) {
4638       // Only the bottom 32 bits matter
4639       cmpw(trial_klass, tmp);
4640       return;
4641     }
4642     decode_klass_not_null(tmp);
4643   } else {
4644     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4645   }
4646   cmp(trial_klass, tmp);
4647 }
4648 
4649 void MacroAssembler::load_prototype_header(Register dst, Register src) {
4650   load_klass(dst, src);
4651   ldr(dst, Address(dst, Klass::prototype_header_offset()));
4652 }
4653 
4654 void MacroAssembler::store_klass(Register dst, Register src) {
4655   // FIXME: Should this be a store release?  concurrent gcs assumes
4656   // klass length is valid if klass field is not null.
4657   if (UseCompressedClassPointers) {
4658     encode_klass_not_null(src);
4659     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4660   } else {
4661     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4662   }
4663 }
4664 
4665 void MacroAssembler::store_klass_gap(Register dst, Register src) {
4666   if (UseCompressedClassPointers) {
4667     // Store to klass gap in destination
4668     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
4669   }
4670 }
4671 
4672 // Algorithm must match CompressedOops::encode.
4673 void MacroAssembler::encode_heap_oop(Register d, Register s) {

4958   if (as_raw) {
4959     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2);
4960   } else {
4961     bs->load_at(this, decorators, type, dst, src, tmp1, tmp2);
4962   }
4963 }
4964 
4965 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
4966                                      Address dst, Register val,
4967                                      Register tmp1, Register tmp2, Register tmp3) {
4968   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4969   decorators = AccessInternal::decorator_fixup(decorators, type);
4970   bool as_raw = (decorators & AS_RAW) != 0;
4971   if (as_raw) {
4972     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
4973   } else {
4974     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
4975   }
4976 }
4977 
4978 void MacroAssembler::access_value_copy(DecoratorSet decorators, Register src, Register dst,
4979                                        Register inline_klass) {
4980   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4981   bs->value_copy(this, decorators, src, dst, inline_klass);
4982 }
4983 
4984 void MacroAssembler::first_field_offset(Register inline_klass, Register offset) {
4985   ldr(offset, Address(inline_klass, InstanceKlass::adr_inlineklass_fixed_block_offset()));
4986   ldrw(offset, Address(offset, InlineKlass::first_field_offset_offset()));
4987 }
4988 
4989 void MacroAssembler::data_for_oop(Register oop, Register data, Register inline_klass) {
4990   // ((address) (void*) o) + vk->first_field_offset();
4991   Register offset = (data == oop) ? rscratch1 : data;
4992   first_field_offset(inline_klass, offset);
4993   if (data == oop) {
4994     add(data, data, offset);
4995   } else {
4996     lea(data, Address(oop, offset));
4997   }
4998 }
4999 
5000 void MacroAssembler::data_for_value_array_index(Register array, Register array_klass,
5001                                                 Register index, Register data) {
5002   assert_different_registers(array, array_klass, index);
5003   assert_different_registers(rscratch1, array, index);
5004 
5005   // array->base() + (index << Klass::layout_helper_log2_element_size(lh));
5006   ldrw(rscratch1, Address(array_klass, Klass::layout_helper_offset()));
5007 
5008   // Klass::layout_helper_log2_element_size(lh)
5009   // (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask;
5010   lsr(rscratch1, rscratch1, Klass::_lh_log2_element_size_shift);
5011   andr(rscratch1, rscratch1, Klass::_lh_log2_element_size_mask);
5012   lslv(index, index, rscratch1);
5013 
5014   add(data, array, index);
5015   add(data, data, arrayOopDesc::base_offset_in_bytes(T_PRIMITIVE_OBJECT));
5016 }
5017 
5018 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
5019                                    Register tmp2, DecoratorSet decorators) {
5020   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
5021 }
5022 
5023 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
5024                                             Register tmp2, DecoratorSet decorators) {
5025   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, tmp2);
5026 }
5027 
5028 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
5029                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
5030   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
5031 }
5032 
5033 // Used for storing nulls.
5034 void MacroAssembler::store_heap_oop_null(Address dst) {
5035   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
5036 }
5037 

5074     oop_index = oop_recorder()->allocate_metadata_index(obj);
5075   } else {
5076     oop_index = oop_recorder()->find_index(obj);
5077   }
5078   RelocationHolder rspec = metadata_Relocation::spec(oop_index);
5079   mov(dst, Address((address)obj, rspec));
5080 }
5081 
5082 Address MacroAssembler::constant_oop_address(jobject obj) {
5083 #ifdef ASSERT
5084   {
5085     ThreadInVMfromUnknown tiv;
5086     assert(oop_recorder() != nullptr, "this assembler needs an OopRecorder");
5087     assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "not an oop");
5088   }
5089 #endif
5090   int oop_index = oop_recorder()->find_index(obj);
5091   return Address((address)obj, oop_Relocation::spec(oop_index));
5092 }
5093 
5094 // Object / value buffer allocation...
5095 void MacroAssembler::allocate_instance(Register klass, Register new_obj,
5096                                        Register t1, Register t2,
5097                                        bool clear_fields, Label& alloc_failed)
5098 {
5099   Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop;
5100   Register layout_size = t1;
5101   assert(new_obj == r0, "needs to be r0");
5102   assert_different_registers(klass, new_obj, t1, t2);
5103 
5104   // get instance_size in InstanceKlass (scaled to a count of bytes)
5105   ldrw(layout_size, Address(klass, Klass::layout_helper_offset()));
5106   // test to see if it has a finalizer or is malformed in some way
5107   tst(layout_size, Klass::_lh_instance_slow_path_bit);
5108   br(Assembler::NE, slow_case_no_pop);
5109 
5110   // Allocate the instance:
5111   //  If TLAB is enabled:
5112   //    Try to allocate in the TLAB.
5113   //    If fails, go to the slow path.
5114   //    Initialize the allocation.
5115   //    Exit.
5116   //
5117   //  Go to slow path.
5118 
5119   if (UseTLAB) {
5120     push(klass);
5121     tlab_allocate(new_obj, layout_size, 0, klass, t2, slow_case);
5122     if (ZeroTLAB || (!clear_fields)) {
5123       // the fields have been already cleared
5124       b(initialize_header);
5125     } else {
5126       // initialize both the header and fields
5127       b(initialize_object);
5128     }
5129 
5130     if (clear_fields) {
5131       // The object is initialized before the header.  If the object size is
5132       // zero, go directly to the header initialization.
5133       bind(initialize_object);
5134       subs(layout_size, layout_size, sizeof(oopDesc));
5135       br(Assembler::EQ, initialize_header);
5136 
5137       // Initialize topmost object field, divide size by 8, check if odd and
5138       // test if zero.
5139 
5140   #ifdef ASSERT
5141       // make sure instance_size was multiple of 8
5142       Label L;
5143       tst(layout_size, 7);
5144       br(Assembler::EQ, L);
5145       stop("object size is not multiple of 8 - adjust this code");
5146       bind(L);
5147       // must be > 0, no extra check needed here
5148   #endif
5149 
5150       lsr(layout_size, layout_size, LogBytesPerLong);
5151 
5152       // initialize remaining object fields: instance_size was a multiple of 8
5153       {
5154         Label loop;
5155         Register base = t2;
5156 
5157         bind(loop);
5158         add(rscratch1, new_obj, layout_size, Assembler::LSL, LogBytesPerLong);
5159         str(zr, Address(rscratch1, sizeof(oopDesc) - 1*oopSize));
5160         subs(layout_size, layout_size, 1);
5161         br(Assembler::NE, loop);
5162       }
5163     } // clear_fields
5164 
5165     // initialize object header only.
5166     bind(initialize_header);
5167     pop(klass);
5168     Register mark_word = t2;
5169     ldr(mark_word, Address(klass, Klass::prototype_header_offset()));
5170     str(mark_word, Address(new_obj, oopDesc::mark_offset_in_bytes ()));
5171     store_klass_gap(new_obj, zr);  // zero klass gap for compressed oops
5172     mov(t2, klass);         // preserve klass
5173     store_klass(new_obj, t2);  // src klass reg is potentially compressed
5174 
5175     // TODO: Valhalla removed SharedRuntime::dtrace_object_alloc from here ?
5176 
5177     b(done);
5178   }
5179 
5180   if (UseTLAB) {
5181     bind(slow_case);
5182     pop(klass);
5183   }
5184   bind(slow_case_no_pop);
5185   b(alloc_failed);
5186 
5187   bind(done);
5188 }
5189 
5190 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
5191 void MacroAssembler::tlab_allocate(Register obj,
5192                                    Register var_size_in_bytes,
5193                                    int con_size_in_bytes,
5194                                    Register t1,
5195                                    Register t2,
5196                                    Label& slow_case) {
5197   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
5198   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
5199 }
5200 
5201 void MacroAssembler::verify_tlab() {
5202 #ifdef ASSERT
5203   if (UseTLAB && VerifyOops) {
5204     Label next, ok;
5205 
5206     stp(rscratch2, rscratch1, Address(pre(sp, -16)));
5207 
5208     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
5209     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
5210     cmp(rscratch2, rscratch1);
5211     br(Assembler::HS, next);
5212     STOP("assert(top >= start)");
5213     should_not_reach_here();
5214 
5215     bind(next);
5216     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
5217     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
5218     cmp(rscratch2, rscratch1);
5219     br(Assembler::HS, ok);
5220     STOP("assert(top <= end)");
5221     should_not_reach_here();
5222 
5223     bind(ok);
5224     ldp(rscratch2, rscratch1, Address(post(sp, 16)));
5225   }
5226 #endif
5227 }
5228 
5229 void MacroAssembler::get_inline_type_field_klass(Register klass, Register index, Register inline_klass) {
5230   ldr(inline_klass, Address(klass, InstanceKlass::inline_type_field_klasses_offset()));
5231 #ifdef ASSERT
5232   {
5233     Label done;
5234     cbnz(inline_klass, done);
5235     stop("get_inline_type_field_klass contains no inline klass");
5236     bind(done);
5237   }
5238 #endif
5239   ldr(inline_klass, Address(inline_klass, index, Address::lsl(3)));
5240 }
5241 
5242 // Writes to stack successive pages until offset reached to check for
5243 // stack overflow + shadow pages.  This clobbers tmp.
5244 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
5245   assert_different_registers(tmp, size, rscratch1);
5246   mov(tmp, sp);
5247   // Bang stack for total size given plus shadow page size.
5248   // Bang one page at a time because large size can bang beyond yellow and
5249   // red zones.
5250   Label loop;
5251   mov(rscratch1, (int)os::vm_page_size());
5252   bind(loop);
5253   lea(tmp, Address(tmp, -(int)os::vm_page_size()));
5254   subsw(size, size, rscratch1);
5255   str(size, Address(tmp));
5256   br(Assembler::GT, loop);
5257 
5258   // Bang down shadow pages too.
5259   // At this point, (tmp-0) is the last address touched, so don't
5260   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
5261   // was post-decremented.)  Skip this address by starting at i=1, and

5347 }
5348 
5349 void MacroAssembler::remove_frame(int framesize) {
5350   assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR");
5351   assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
5352   if (framesize < ((1 << 9) + 2 * wordSize)) {
5353     ldp(rfp, lr, Address(sp, framesize - 2 * wordSize));
5354     add(sp, sp, framesize);
5355   } else {
5356     if (framesize < ((1 << 12) + 2 * wordSize))
5357       add(sp, sp, framesize - 2 * wordSize);
5358     else {
5359       mov(rscratch1, framesize - 2 * wordSize);
5360       add(sp, sp, rscratch1);
5361     }
5362     ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
5363   }
5364   authenticate_return_address();
5365 }
5366 
5367 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
5368   if (needs_stack_repair) {
5369     // Remove the extension of the caller's frame used for inline type unpacking
5370     //
5371     // Right now the stack looks like this:
5372     //
5373     // | Arguments from caller     |
5374     // |---------------------------|  <-- caller's SP
5375     // | Saved LR #1               |
5376     // | Saved FP #1               |
5377     // |---------------------------|
5378     // | Extension space for       |
5379     // |   inline arg (un)packing  |
5380     // |---------------------------|  <-- start of this method's frame
5381     // | Saved LR #2               |
5382     // | Saved FP #2               |
5383     // |---------------------------|  <-- FP
5384     // | sp_inc                    |
5385     // | method locals             |
5386     // |---------------------------|  <-- SP
5387     //
5388     // There are two copies of FP and LR on the stack. They will be identical
5389     // unless the caller has been deoptimized, in which case LR #1 will be patched
5390     // to point at the deopt blob, and LR #2 will still point into the old method.
5391     //
5392     // The sp_inc stack slot holds the total size of the frame including the
5393     // extension space minus two words for the saved FP and LR.
5394 
5395     int sp_inc_offset = initial_framesize - 3 * wordSize;  // Immediately below saved LR and FP
5396 
5397     ldr(rscratch1, Address(sp, sp_inc_offset));
5398     add(sp, sp, rscratch1);
5399     ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
5400   } else {
5401     remove_frame(initial_framesize);
5402   }
5403 }
5404 
5405 void MacroAssembler::save_stack_increment(int sp_inc, int frame_size) {
5406   int real_frame_size = frame_size + sp_inc;
5407   assert(sp_inc == 0 || sp_inc > 2*wordSize, "invalid sp_inc value");
5408   assert(real_frame_size >= 2*wordSize, "frame size must include FP/LR space");
5409   assert((real_frame_size & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
5410 
5411   int sp_inc_offset = frame_size - 3 * wordSize;  // Immediately below saved LR and FP
5412 
5413   // Subtract two words for the saved FP and LR as these will be popped
5414   // separately. See remove_frame above.
5415   mov(rscratch1, real_frame_size - 2*wordSize);
5416   str(rscratch1, Address(sp, sp_inc_offset));
5417 }
5418 
5419 // This method counts leading positive bytes (highest bit not set) in provided byte array
5420 address MacroAssembler::count_positives(Register ary1, Register len, Register result) {
5421     // Simple and most common case of aligned small array which is not at the
5422     // end of memory page is placed here. All other cases are in stub.
5423     Label LOOP, END, STUB, STUB_LONG, SET_RESULT, DONE;
5424     const uint64_t UPPER_BIT_MASK=0x8080808080808080;
5425     assert_different_registers(ary1, len, result);
5426 
5427     mov(result, len);
5428     cmpw(len, 0);
5429     br(LE, DONE);
5430     cmpw(len, 4 * wordSize);
5431     br(GE, STUB_LONG); // size > 32 then go to stub
5432 
5433     int shift = 64 - exact_log2(os::vm_page_size());
5434     lsl(rscratch1, ary1, shift);
5435     mov(rscratch2, (size_t)(4 * wordSize) << shift);
5436     adds(rscratch2, rscratch1, rscratch2);  // At end of page?
5437     br(CS, STUB); // at the end of page then go to stub

6312 // On other systems, the helper is a usual C function.
6313 //
6314 void MacroAssembler::get_thread(Register dst) {
6315   RegSet saved_regs =
6316     LINUX_ONLY(RegSet::range(r0, r1)  + lr - dst)
6317     NOT_LINUX (RegSet::range(r0, r17) + lr - dst);
6318 
6319   protect_return_address();
6320   push(saved_regs, sp);
6321 
6322   mov(lr, CAST_FROM_FN_PTR(address, JavaThread::aarch64_get_thread_helper));
6323   blr(lr);
6324   if (dst != c_rarg0) {
6325     mov(dst, c_rarg0);
6326   }
6327 
6328   pop(saved_regs, sp);
6329   authenticate_return_address();
6330 }
6331 
6332 #ifdef COMPILER2
6333 // C2 compiled method's prolog code
6334 // Moved here from aarch64.ad to support Valhalla code belows
6335 void MacroAssembler::verified_entry(Compile* C, int sp_inc) {
6336   if (C->clinit_barrier_on_entry()) {
6337     assert(!C->method()->holder()->is_not_initialized(), "initialization should have been started");
6338 
6339     Label L_skip_barrier;
6340 
6341     mov_metadata(rscratch2, C->method()->holder()->constant_encoding());
6342     clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
6343     far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
6344     bind(L_skip_barrier);
6345   }
6346 
6347   if (C->max_vector_size() > 0) {
6348     reinitialize_ptrue();
6349   }
6350 
6351   int bangsize = C->output()->bang_size_in_bytes();
6352   if (C->output()->need_stack_bang(bangsize))
6353     generate_stack_overflow_check(bangsize);
6354 
6355   // n.b. frame size includes space for return pc and rfp
6356   const long framesize = C->output()->frame_size_in_bytes();
6357   build_frame(framesize);
6358 
6359   if (C->needs_stack_repair()) {
6360     save_stack_increment(sp_inc, framesize);
6361   }
6362 
6363   if (VerifyStackAtCalls) {
6364     Unimplemented();
6365   }
6366 }
6367 #endif // COMPILER2
6368 
6369 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
6370   assert(InlineTypeReturnedAsFields, "Inline types should never be returned as fields");
6371   // An inline type might be returned. If fields are in registers we
6372   // need to allocate an inline type instance and initialize it with
6373   // the value of the fields.
6374   Label skip;
6375   // We only need a new buffered inline type if a new one is not returned
6376   tbz(r0, 0, skip);
6377   int call_offset = -1;
6378 
6379   // Be careful not to clobber r1-7 which hold returned fields
6380   // Also do not use callee-saved registers as these may be live in the interpreter
6381   Register tmp1 = r13, tmp2 = r14, klass = r15, r0_preserved = r12;
6382 
6383   // The following code is similar to allocate_instance but has some slight differences,
6384   // e.g. object size is always not zero, sometimes it's constant; storing klass ptr after
6385   // allocating is not necessary if vk != nullptr, etc. allocate_instance is not aware of these.
6386   Label slow_case;
6387   // 1. Try to allocate a new buffered inline instance either from TLAB or eden space
6388   mov(r0_preserved, r0); // save r0 for slow_case since *_allocate may corrupt it when allocation failed
6389 
6390   if (vk != nullptr) {
6391     // Called from C1, where the return type is statically known.
6392     movptr(klass, (intptr_t)vk->get_InlineKlass());
6393     jint obj_size = vk->layout_helper();
6394     assert(obj_size != Klass::_lh_neutral_value, "inline class in return type must have been resolved");
6395     if (UseTLAB) {
6396       tlab_allocate(r0, noreg, obj_size, tmp1, tmp2, slow_case);
6397     } else {
6398       b(slow_case);
6399     }
6400   } else {
6401     // Call from interpreter. R0 contains ((the InlineKlass* of the return type) | 0x01)
6402     andr(klass, r0, -2);
6403     ldrw(tmp2, Address(klass, Klass::layout_helper_offset()));
6404     if (UseTLAB) {
6405       tlab_allocate(r0, tmp2, 0, tmp1, tmp2, slow_case);
6406     } else {
6407       b(slow_case);
6408     }
6409   }
6410   if (UseTLAB) {
6411     // 2. Initialize buffered inline instance header
6412     Register buffer_obj = r0;
6413     mov(rscratch1, (intptr_t)markWord::inline_type_prototype().value());
6414     str(rscratch1, Address(buffer_obj, oopDesc::mark_offset_in_bytes()));
6415     store_klass_gap(buffer_obj, zr);
6416     if (vk == nullptr) {
6417       // store_klass corrupts klass, so save it for later use (interpreter case only).
6418       mov(tmp1, klass);
6419     }
6420     store_klass(buffer_obj, klass);
6421     // 3. Initialize its fields with an inline class specific handler
6422     if (vk != nullptr) {
6423       far_call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint.
6424     } else {
6425       // tmp1 holds klass preserved above
6426       ldr(tmp1, Address(tmp1, InstanceKlass::adr_inlineklass_fixed_block_offset()));
6427       ldr(tmp1, Address(tmp1, InlineKlass::pack_handler_offset()));
6428       blr(tmp1);
6429     }
6430 
6431     membar(Assembler::StoreStore);
6432     b(skip);
6433   } else {
6434     // Must have already branched to slow_case above.
6435     DEBUG_ONLY(should_not_reach_here());
6436   }
6437   bind(slow_case);
6438   // We failed to allocate a new inline type, fall back to a runtime
6439   // call. Some oop field may be live in some registers but we can't
6440   // tell. That runtime call will take care of preserving them
6441   // across a GC if there's one.
6442   mov(r0, r0_preserved);
6443 
6444   if (from_interpreter) {
6445     super_call_VM_leaf(StubRoutines::store_inline_type_fields_to_buf());
6446   } else {
6447     far_call(RuntimeAddress(StubRoutines::store_inline_type_fields_to_buf()));
6448     call_offset = offset();
6449   }
6450   membar(Assembler::StoreStore);
6451 
6452   bind(skip);
6453   return call_offset;
6454 }
6455 
6456 // Move a value between registers/stack slots and update the reg_state
6457 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
6458   assert(from->is_valid() && to->is_valid(), "source and destination must be valid");
6459   if (reg_state[to->value()] == reg_written) {
6460     return true; // Already written
6461   }
6462 
6463   if (from != to && bt != T_VOID) {
6464     if (reg_state[to->value()] == reg_readonly) {
6465       return false; // Not yet writable
6466     }
6467     if (from->is_reg()) {
6468       if (to->is_reg()) {
6469         if (from->is_Register() && to->is_Register()) {
6470           mov(to->as_Register(), from->as_Register());
6471         } else if (from->is_FloatRegister() && to->is_FloatRegister()) {
6472           fmovd(to->as_FloatRegister(), from->as_FloatRegister());
6473         } else {
6474           ShouldNotReachHere();
6475         }
6476       } else {
6477         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size;
6478         Address to_addr = Address(sp, st_off);
6479         if (from->is_FloatRegister()) {
6480           if (bt == T_DOUBLE) {
6481              strd(from->as_FloatRegister(), to_addr);
6482           } else {
6483              assert(bt == T_FLOAT, "must be float");
6484              strs(from->as_FloatRegister(), to_addr);
6485           }
6486         } else {
6487           str(from->as_Register(), to_addr);
6488         }
6489       }
6490     } else {
6491       Address from_addr = Address(sp, from->reg2stack() * VMRegImpl::stack_slot_size);
6492       if (to->is_reg()) {
6493         if (to->is_FloatRegister()) {
6494           if (bt == T_DOUBLE) {
6495             ldrd(to->as_FloatRegister(), from_addr);
6496           } else {
6497             assert(bt == T_FLOAT, "must be float");
6498             ldrs(to->as_FloatRegister(), from_addr);
6499           }
6500         } else {
6501           ldr(to->as_Register(), from_addr);
6502         }
6503       } else {
6504         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size;
6505         ldr(rscratch1, from_addr);
6506         str(rscratch1, Address(sp, st_off));
6507       }
6508     }
6509   }
6510 
6511   // Update register states
6512   reg_state[from->value()] = reg_writable;
6513   reg_state[to->value()] = reg_written;
6514   return true;
6515 }
6516 
6517 // Calculate the extra stack space required for packing or unpacking inline
6518 // args and adjust the stack pointer
6519 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
6520   int sp_inc = args_on_stack * VMRegImpl::stack_slot_size;
6521   sp_inc = align_up(sp_inc, StackAlignmentInBytes);
6522   assert(sp_inc > 0, "sanity");
6523 
6524   // Save a copy of the FP and LR here for deoptimization patching and frame walking
6525   stp(rfp, lr, Address(pre(sp, -2 * wordSize)));
6526 
6527   // Adjust the stack pointer. This will be repaired on return by MacroAssembler::remove_frame
6528   if (sp_inc < (1 << 9)) {
6529     sub(sp, sp, sp_inc);   // Fits in an immediate
6530   } else {
6531     mov(rscratch1, sp_inc);
6532     sub(sp, sp, rscratch1);
6533   }
6534 
6535   return sp_inc + 2 * wordSize;  // Account for the FP/LR space
6536 }
6537 
6538 // Read all fields from an inline type oop and store the values in registers/stack slots
6539 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
6540                                           VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
6541                                           RegState reg_state[]) {
6542   assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter");
6543   assert(from->is_valid(), "source must be valid");
6544   bool progress = false;
6545 #ifdef ASSERT
6546   const int start_offset = offset();
6547 #endif
6548 
6549   Label L_null, L_notNull;
6550   // Don't use r14 as tmp because it's used for spilling (see MacroAssembler::spill_reg_for)
6551   Register tmp1 = r10;
6552   Register tmp2 = r11;
6553   Register fromReg = noreg;
6554   ScalarizedInlineArgsStream stream(sig, sig_index, to, to_count, to_index, -1);
6555   bool done = true;
6556   bool mark_done = true;
6557   VMReg toReg;
6558   BasicType bt;
6559   // Check if argument requires a null check
6560   bool null_check = false;
6561   VMReg nullCheckReg;
6562   while (stream.next(nullCheckReg, bt)) {
6563     if (sig->at(stream.sig_index())._offset == -1) {
6564       null_check = true;
6565       break;
6566     }
6567   }
6568   stream.reset(sig_index, to_index);
6569   while (stream.next(toReg, bt)) {
6570     assert(toReg->is_valid(), "destination must be valid");
6571     int idx = (int)toReg->value();
6572     if (reg_state[idx] == reg_readonly) {
6573       if (idx != from->value()) {
6574         mark_done = false;
6575       }
6576       done = false;
6577       continue;
6578     } else if (reg_state[idx] == reg_written) {
6579       continue;
6580     }
6581     assert(reg_state[idx] == reg_writable, "must be writable");
6582     reg_state[idx] = reg_written;
6583     progress = true;
6584 
6585     if (fromReg == noreg) {
6586       if (from->is_reg()) {
6587         fromReg = from->as_Register();
6588       } else {
6589         int st_off = from->reg2stack() * VMRegImpl::stack_slot_size;
6590         ldr(tmp1, Address(sp, st_off));
6591         fromReg = tmp1;
6592       }
6593       if (null_check) {
6594         // Nullable inline type argument, emit null check
6595         cbz(fromReg, L_null);
6596       }
6597     }
6598     int off = sig->at(stream.sig_index())._offset;
6599     if (off == -1) {
6600       assert(null_check, "Missing null check at");
6601       if (toReg->is_stack()) {
6602         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size;
6603         mov(tmp2, 1);
6604         str(tmp2, Address(sp, st_off));
6605       } else {
6606         mov(toReg->as_Register(), 1);
6607       }
6608       continue;
6609     }
6610     assert(off > 0, "offset in object should be positive");
6611     Address fromAddr = Address(fromReg, off);
6612     if (!toReg->is_FloatRegister()) {
6613       Register dst = toReg->is_stack() ? tmp2 : toReg->as_Register();
6614       if (is_reference_type(bt)) {
6615         load_heap_oop(dst, fromAddr, rscratch1, rscratch2);
6616       } else {
6617         bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN);
6618         load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed);
6619       }
6620       if (toReg->is_stack()) {
6621         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size;
6622         str(dst, Address(sp, st_off));
6623       }
6624     } else if (bt == T_DOUBLE) {
6625       ldrd(toReg->as_FloatRegister(), fromAddr);
6626     } else {
6627       assert(bt == T_FLOAT, "must be float");
6628       ldrs(toReg->as_FloatRegister(), fromAddr);
6629     }
6630   }
6631   if (progress && null_check) {
6632     if (done) {
6633       b(L_notNull);
6634       bind(L_null);
6635       // Set IsInit field to zero to signal that the argument is null.
6636       // Also set all oop fields to zero to make the GC happy.
6637       stream.reset(sig_index, to_index);
6638       while (stream.next(toReg, bt)) {
6639         if (sig->at(stream.sig_index())._offset == -1 ||
6640             bt == T_OBJECT || bt == T_ARRAY) {
6641           if (toReg->is_stack()) {
6642             int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size;
6643             str(zr, Address(sp, st_off));
6644           } else {
6645             mov(toReg->as_Register(), zr);
6646           }
6647         }
6648       }
6649       bind(L_notNull);
6650     } else {
6651       bind(L_null);
6652     }
6653   }
6654 
6655   sig_index = stream.sig_index();
6656   to_index = stream.regs_index();
6657 
6658   if (mark_done && reg_state[from->value()] != reg_written) {
6659     // This is okay because no one else will write to that slot
6660     reg_state[from->value()] = reg_writable;
6661   }
6662   from_index--;
6663   assert(progress || (start_offset == offset()), "should not emit code");
6664   return done;
6665 }
6666 
6667 // Pack fields back into an inline type oop
6668 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
6669                                         VMRegPair* from, int from_count, int& from_index, VMReg to,
6670                                         RegState reg_state[], Register val_array) {
6671   assert(sig->at(sig_index)._bt == T_METADATA, "should be at delimiter");
6672   assert(to->is_valid(), "destination must be valid");
6673 
6674   if (reg_state[to->value()] == reg_written) {
6675     skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
6676     return true; // Already written
6677   }
6678 
6679   // The GC barrier expanded by store_heap_oop below may call into the
6680   // runtime so use callee-saved registers for any values that need to be
6681   // preserved. The GC barrier assembler should take care of saving the
6682   // Java argument registers.
6683   // TODO 8284443 Isn't it an issue if below code uses r14 as tmp when it contains a spilled value?
6684   // Be careful with r14 because it's used for spilling (see MacroAssembler::spill_reg_for).
6685   Register val_obj_tmp = r21;
6686   Register from_reg_tmp = r22;
6687   Register tmp1 = r14;
6688   Register tmp2 = r13;
6689   Register tmp3 = r12;
6690   Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register();
6691 
6692   assert_different_registers(val_obj_tmp, from_reg_tmp, tmp1, tmp2, tmp3, val_array);
6693 
6694   if (reg_state[to->value()] == reg_readonly) {
6695     if (!is_reg_in_unpacked_fields(sig, sig_index, to, from, from_count, from_index)) {
6696       skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
6697       return false; // Not yet writable
6698     }
6699     val_obj = val_obj_tmp;
6700   }
6701 
6702   int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_OBJECT);
6703   load_heap_oop(val_obj, Address(val_array, index), tmp1, tmp2);
6704 
6705   ScalarizedInlineArgsStream stream(sig, sig_index, from, from_count, from_index);
6706   VMReg fromReg;
6707   BasicType bt;
6708   Label L_null;
6709   while (stream.next(fromReg, bt)) {
6710     assert(fromReg->is_valid(), "source must be valid");
6711     reg_state[fromReg->value()] = reg_writable;
6712 
6713     int off = sig->at(stream.sig_index())._offset;
6714     if (off == -1) {
6715       // Nullable inline type argument, emit null check
6716       Label L_notNull;
6717       if (fromReg->is_stack()) {
6718         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size;
6719         ldrb(tmp2, Address(sp, ld_off));
6720         cbnz(tmp2, L_notNull);
6721       } else {
6722         cbnz(fromReg->as_Register(), L_notNull);
6723       }
6724       mov(val_obj, 0);
6725       b(L_null);
6726       bind(L_notNull);
6727       continue;
6728     }
6729 
6730     assert(off > 0, "offset in object should be positive");
6731     size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
6732 
6733     // Pack the scalarized field into the value object.
6734     Address dst(val_obj, off);
6735 
6736     if (!fromReg->is_FloatRegister()) {
6737       Register src;
6738       if (fromReg->is_stack()) {
6739         src = from_reg_tmp;
6740         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size;
6741         load_sized_value(src, Address(sp, ld_off), size_in_bytes, /* is_signed */ false);
6742       } else {
6743         src = fromReg->as_Register();
6744       }
6745       assert_different_registers(dst.base(), src, tmp1, tmp2, tmp3, val_array);
6746       if (is_reference_type(bt)) {
6747         store_heap_oop(dst, src, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
6748       } else {
6749         store_sized_value(dst, src, size_in_bytes);
6750       }
6751     } else if (bt == T_DOUBLE) {
6752       strd(fromReg->as_FloatRegister(), dst);
6753     } else {
6754       assert(bt == T_FLOAT, "must be float");
6755       strs(fromReg->as_FloatRegister(), dst);
6756     }
6757   }
6758   bind(L_null);
6759   sig_index = stream.sig_index();
6760   from_index = stream.regs_index();
6761 
6762   assert(reg_state[to->value()] == reg_writable, "must have already been read");
6763   bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state);
6764   assert(success, "to register must be writeable");
6765 
6766   return true;
6767 }
6768 
6769 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
6770   return (reg->is_FloatRegister()) ? v8->as_VMReg() : r14->as_VMReg();
6771 }
6772 
6773 void MacroAssembler::cache_wb(Address line) {
6774   assert(line.getMode() == Address::base_plus_offset, "mode should be base_plus_offset");
6775   assert(line.index() == noreg, "index should be noreg");
6776   assert(line.offset() == 0, "offset should be 0");
6777   // would like to assert this
6778   // assert(line._ext.shift == 0, "shift should be zero");
6779   if (VM_Version::supports_dcpop()) {
6780     // writeback using clear virtual address to point of persistence
6781     dc(Assembler::CVAP, line.base());
6782   } else {
6783     // no need to generate anything as Unsafe.writebackMemory should
6784     // never invoke this stub
6785   }
6786 }
6787 
6788 void MacroAssembler::cache_wbsync(bool is_pre) {
6789   // we only need a barrier post sync
6790   if (!is_pre) {
6791     membar(Assembler::AnyAny);
6792   }

7126 }
7127 
7128 // Implements lightweight-locking.
7129 // Branches to slow upon failure to lock the object, with ZF cleared.
7130 // Falls through upon success with ZF set.
7131 //
7132 //  - obj: the object to be locked
7133 //  - hdr: the header, already loaded from obj, will be destroyed
7134 //  - t1, t2: temporary registers, will be destroyed
7135 void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register t1, Register t2, Label& slow) {
7136   assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
7137   assert_different_registers(obj, hdr, t1, t2, rscratch1);
7138 
7139   // Check if we would have space on lock-stack for the object.
7140   ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
7141   cmpw(t1, (unsigned)LockStack::end_offset() - 1);
7142   br(Assembler::GT, slow);
7143 
7144   // Load (object->mark() | 1) into hdr
7145   orr(hdr, hdr, markWord::unlocked_value);
7146   if (EnableValhalla) {
7147     // Mask inline_type bit such that we go to the slow path if object is an inline type
7148     andr(hdr, hdr, ~((int) markWord::inline_type_bit_in_place));
7149   }
7150 
7151   // Clear lock-bits, into t2
7152   eor(t2, hdr, markWord::unlocked_value);
7153   // Try to swing header from unlocked to locked
7154   // Clobbers rscratch1 when UseLSE is false
7155   cmpxchg(/*addr*/ obj, /*expected*/ hdr, /*new*/ t2, Assembler::xword,
7156           /*acquire*/ true, /*release*/ true, /*weak*/ false, t1);
7157   br(Assembler::NE, slow);
7158 
7159   // After successful lock, push object on lock-stack
7160   ldrw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
7161   str(obj, Address(rthread, t1));
7162   addw(t1, t1, oopSize);
7163   strw(t1, Address(rthread, JavaThread::lock_stack_top_offset()));
7164 }
7165 
7166 // Implements lightweight-unlocking.
7167 // Branches to slow upon failure, with ZF cleared.
7168 // Falls through upon success, with ZF set.
7169 //
7170 // - obj: the object to be unlocked
< prev index next >