< prev index next >

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

Print this page

  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include <sys/types.h>
  27 
  28 #include "precompiled.hpp"
  29 #include "jvm.h"
  30 #include "asm/assembler.hpp"
  31 #include "asm/assembler.inline.hpp"
  32 #include "ci/ciEnv.hpp"

  33 #include "compiler/oopMap.hpp"
  34 #include "gc/shared/barrierSet.hpp"
  35 #include "gc/shared/barrierSetAssembler.hpp"
  36 #include "gc/shared/cardTableBarrierSet.hpp"
  37 #include "gc/shared/cardTable.hpp"
  38 #include "gc/shared/collectedHeap.hpp"
  39 #include "gc/shared/tlab_globals.hpp"
  40 #include "interpreter/bytecodeHistogram.hpp"
  41 #include "interpreter/interpreter.hpp"
  42 #include "compiler/compileTask.hpp"
  43 #include "compiler/disassembler.hpp"
  44 #include "memory/resourceArea.hpp"
  45 #include "memory/universe.hpp"
  46 #include "nativeInst_aarch64.hpp"
  47 #include "oops/accessDecorators.hpp"
  48 #include "oops/compressedOops.inline.hpp"
  49 #include "oops/klass.inline.hpp"
  50 #include "runtime/continuation.hpp"
  51 #include "runtime/icache.hpp"
  52 #include "runtime/interfaceSupport.inline.hpp"
  53 #include "runtime/javaThread.hpp"
  54 #include "runtime/jniHandles.inline.hpp"
  55 #include "runtime/sharedRuntime.hpp"

  56 #include "runtime/stubRoutines.hpp"
  57 #include "utilities/powerOfTwo.hpp"

  58 #ifdef COMPILER1
  59 #include "c1/c1_LIRAssembler.hpp"
  60 #endif
  61 #ifdef COMPILER2
  62 #include "oops/oop.hpp"
  63 #include "opto/compile.hpp"
  64 #include "opto/node.hpp"
  65 #include "opto/output.hpp"
  66 #endif
  67 
  68 #ifdef PRODUCT
  69 #define BLOCK_COMMENT(str) /* nothing */
  70 #else
  71 #define BLOCK_COMMENT(str) block_comment(str)
  72 #endif
  73 #define STOP(str) stop(str);
  74 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  75 
  76 #ifdef ASSERT
  77 extern "C" void disnm(intptr_t p);

1095   while (offset() % modulus != 0) nop();
1096 }
1097 
1098 void MacroAssembler::post_call_nop() {
1099   if (!Continuations::enabled()) {
1100     return;
1101   }
1102   InstructionMark im(this);
1103   relocate(post_call_nop_Relocation::spec());
1104   nop();
1105   movk(zr, 0);
1106   movk(zr, 0);
1107 }
1108 
1109 // these are no-ops overridden by InterpreterMacroAssembler
1110 
1111 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { }
1112 
1113 void MacroAssembler::check_and_handle_popframe(Register java_thread) { }
1114 



































1115 // Look up the method for a megamorphic invokeinterface call.
1116 // The target method is determined by <intf_klass, itable_index>.
1117 // The receiver klass is in recv_klass.
1118 // On success, the result will be in method_result, and execution falls through.
1119 // On failure, execution transfers to the given label.
1120 void MacroAssembler::lookup_interface_method(Register recv_klass,
1121                                              Register intf_klass,
1122                                              RegisterOrConstant itable_index,
1123                                              Register method_result,
1124                                              Register scan_temp,
1125                                              Label& L_no_such_interface,
1126                          bool return_method) {
1127   assert_different_registers(recv_klass, intf_klass, scan_temp);
1128   assert_different_registers(method_result, intf_klass, scan_temp);
1129   assert(recv_klass != method_result || !return_method,
1130      "recv_klass can be destroyed when method isn't needed");
1131   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
1132          "caller must use same register for non-constant itable index as for method");
1133 
1134   // Compute start of first itableOffsetEntry (which is at the end of the vtable)

1445   ldrb(scratch, Address(klass, InstanceKlass::init_state_offset()));
1446   subs(zr, scratch, InstanceKlass::fully_initialized);
1447   br(Assembler::EQ, *L_fast_path);
1448 
1449   // Fast path check: current thread is initializer thread
1450   ldr(scratch, Address(klass, InstanceKlass::init_thread_offset()));
1451   cmp(rthread, scratch);
1452 
1453   if (L_slow_path == &L_fallthrough) {
1454     br(Assembler::EQ, *L_fast_path);
1455     bind(*L_slow_path);
1456   } else if (L_fast_path == &L_fallthrough) {
1457     br(Assembler::NE, *L_slow_path);
1458     bind(*L_fast_path);
1459   } else {
1460     Unimplemented();
1461   }
1462 }
1463 
1464 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
1465   if (!VerifyOops) return;




1466 
1467   // Pass register number to verify_oop_subroutine
1468   const char* b = NULL;
1469   {
1470     ResourceMark rm;
1471     stringStream ss;
1472     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
1473     b = code_string(ss.as_string());
1474   }
1475   BLOCK_COMMENT("verify_oop {");
1476 
1477   strip_return_address(); // This might happen within a stack frame.
1478   protect_return_address();
1479   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1480   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1481 
1482   mov(r0, reg);
1483   movptr(rscratch1, (uintptr_t)(address)b);
1484 
1485   // call indirectly to solve generation ordering problem
1486   lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1487   ldr(rscratch2, Address(rscratch2));
1488   blr(rscratch2);
1489 
1490   ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1491   ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1492   authenticate_return_address();
1493 
1494   BLOCK_COMMENT("} verify_oop");
1495 }
1496 
1497 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
1498   if (!VerifyOops) return;




1499 
1500   const char* b = NULL;
1501   {
1502     ResourceMark rm;
1503     stringStream ss;
1504     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
1505     b = code_string(ss.as_string());
1506   }
1507   BLOCK_COMMENT("verify_oop_addr {");
1508 
1509   strip_return_address(); // This might happen within a stack frame.
1510   protect_return_address();
1511   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1512   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1513 
1514   // addr may contain sp so we will have to adjust it based on the
1515   // pushes that we just did.
1516   if (addr.uses(sp)) {
1517     lea(r0, addr);
1518     ldr(r0, Address(r0, 4 * wordSize));

1560   stp(rscratch1, rmethod, Address(pre(sp, -2 * wordSize)));
1561 
1562   mov(rscratch1, entry_point);
1563   blr(rscratch1);
1564   if (retaddr)
1565     bind(*retaddr);
1566 
1567   ldp(rscratch1, rmethod, Address(post(sp, 2 * wordSize)));
1568 }
1569 
1570 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
1571   call_VM_leaf_base(entry_point, number_of_arguments);
1572 }
1573 
1574 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
1575   pass_arg0(this, arg_0);
1576   call_VM_leaf_base(entry_point, 1);
1577 }
1578 
1579 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {

1580   pass_arg0(this, arg_0);
1581   pass_arg1(this, arg_1);
1582   call_VM_leaf_base(entry_point, 2);
1583 }
1584 
1585 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
1586                                   Register arg_1, Register arg_2) {


1587   pass_arg0(this, arg_0);
1588   pass_arg1(this, arg_1);
1589   pass_arg2(this, arg_2);
1590   call_VM_leaf_base(entry_point, 3);
1591 }
1592 




1593 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1594   pass_arg0(this, arg_0);
1595   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1596 }
1597 
1598 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1599 
1600   assert(arg_0 != c_rarg1, "smashed arg");
1601   pass_arg1(this, arg_1);
1602   pass_arg0(this, arg_0);
1603   MacroAssembler::call_VM_leaf_base(entry_point, 2);
1604 }
1605 
1606 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1607   assert(arg_0 != c_rarg2, "smashed arg");
1608   assert(arg_1 != c_rarg2, "smashed arg");
1609   pass_arg2(this, arg_2);
1610   assert(arg_0 != c_rarg1, "smashed arg");
1611   pass_arg1(this, arg_1);
1612   pass_arg0(this, arg_0);

1622   assert(arg_1 != c_rarg2, "smashed arg");
1623   pass_arg2(this, arg_2);
1624   assert(arg_0 != c_rarg1, "smashed arg");
1625   pass_arg1(this, arg_1);
1626   pass_arg0(this, arg_0);
1627   MacroAssembler::call_VM_leaf_base(entry_point, 4);
1628 }
1629 
1630 void MacroAssembler::null_check(Register reg, int offset) {
1631   if (needs_explicit_null_check(offset)) {
1632     // provoke OS NULL exception if reg = NULL by
1633     // accessing M[reg] w/o changing any registers
1634     // NOTE: this is plenty to provoke a segv
1635     ldr(zr, Address(reg));
1636   } else {
1637     // nothing to do, (later) access of M[reg + offset]
1638     // will provoke OS NULL exception if reg = NULL
1639   }
1640 }
1641 














































































































1642 // MacroAssembler protected routines needed to implement
1643 // public methods
1644 
1645 void MacroAssembler::mov(Register r, Address dest) {
1646   code_section()->relocate(pc(), dest.rspec());
1647   uint64_t imm64 = (uint64_t)dest.target();
1648   movptr(r, imm64);
1649 }
1650 
1651 // Move a constant pointer into r.  In AArch64 mode the virtual
1652 // address space is 48 bits in size, so we only need three
1653 // instructions to create a patchable instruction sequence that can
1654 // reach anywhere.
1655 void MacroAssembler::movptr(Register r, uintptr_t imm64) {
1656 #ifndef PRODUCT
1657   {
1658     char buffer[64];
1659     snprintf(buffer, sizeof(buffer), "0x%" PRIX64, (uint64_t)imm64);
1660     block_comment(buffer);
1661   }

4024   adrp(rscratch1, src2, offset);
4025   ldr(rscratch1, Address(rscratch1, offset));
4026   cmp(src1, rscratch1);
4027 }
4028 
4029 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
4030   cmp(obj1, obj2);
4031 }
4032 
4033 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4034   load_method_holder(rresult, rmethod);
4035   ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4036 }
4037 
4038 void MacroAssembler::load_method_holder(Register holder, Register method) {
4039   ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
4040   ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
4041   ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4042 }
4043 








4044 void MacroAssembler::load_klass(Register dst, Register src) {
4045   if (UseCompressedClassPointers) {
4046     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4047     decode_klass_not_null(dst);
4048   } else {
4049     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4050   }
4051 }
4052 
4053 // ((OopHandle)result).resolve();
4054 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) {
4055   // OopHandle::resolve is an indirection.
4056   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp1, tmp2);
4057 }
4058 
4059 // ((WeakHandle)result).resolve();
4060 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2) {
4061   assert_different_registers(result, tmp1, tmp2);
4062   Label resolved;
4063 

4082 
4083 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
4084   if (UseCompressedClassPointers) {
4085     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4086     if (CompressedKlassPointers::base() == NULL) {
4087       cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
4088       return;
4089     } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
4090                && CompressedKlassPointers::shift() == 0) {
4091       // Only the bottom 32 bits matter
4092       cmpw(trial_klass, tmp);
4093       return;
4094     }
4095     decode_klass_not_null(tmp);
4096   } else {
4097     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4098   }
4099   cmp(trial_klass, tmp);
4100 }
4101 





4102 void MacroAssembler::store_klass(Register dst, Register src) {
4103   // FIXME: Should this be a store release?  concurrent gcs assumes
4104   // klass length is valid if klass field is not null.
4105   if (UseCompressedClassPointers) {
4106     encode_klass_not_null(src);
4107     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4108   } else {
4109     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4110   }
4111 }
4112 
4113 void MacroAssembler::store_klass_gap(Register dst, Register src) {
4114   if (UseCompressedClassPointers) {
4115     // Store to klass gap in destination
4116     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
4117   }
4118 }
4119 
4120 // Algorithm must match CompressedOops::encode.
4121 void MacroAssembler::encode_heap_oop(Register d, Register s) {

4406   if (as_raw) {
4407     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2);
4408   } else {
4409     bs->load_at(this, decorators, type, dst, src, tmp1, tmp2);
4410   }
4411 }
4412 
4413 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
4414                                      Address dst, Register src,
4415                                      Register tmp1, Register tmp2, Register tmp3) {
4416   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4417   decorators = AccessInternal::decorator_fixup(decorators);
4418   bool as_raw = (decorators & AS_RAW) != 0;
4419   if (as_raw) {
4420     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
4421   } else {
4422     bs->store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
4423   }
4424 }
4425 








































4426 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
4427                                    Register tmp2, DecoratorSet decorators) {
4428   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
4429 }
4430 
4431 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
4432                                             Register tmp2, DecoratorSet decorators) {
4433   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, tmp2);
4434 }
4435 
4436 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
4437                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
4438   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2, tmp3);
4439 }
4440 
4441 // Used for storing NULLs.
4442 void MacroAssembler::store_heap_oop_null(Address dst) {
4443   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
4444 }
4445 

4482     oop_index = oop_recorder()->allocate_metadata_index(obj);
4483   } else {
4484     oop_index = oop_recorder()->find_index(obj);
4485   }
4486   RelocationHolder rspec = metadata_Relocation::spec(oop_index);
4487   mov(dst, Address((address)obj, rspec));
4488 }
4489 
4490 Address MacroAssembler::constant_oop_address(jobject obj) {
4491 #ifdef ASSERT
4492   {
4493     ThreadInVMfromUnknown tiv;
4494     assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
4495     assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "not an oop");
4496   }
4497 #endif
4498   int oop_index = oop_recorder()->find_index(obj);
4499   return Address((address)obj, oop_Relocation::spec(oop_index));
4500 }
4501 
































































































4502 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4503 void MacroAssembler::tlab_allocate(Register obj,
4504                                    Register var_size_in_bytes,
4505                                    int con_size_in_bytes,
4506                                    Register t1,
4507                                    Register t2,
4508                                    Label& slow_case) {
4509   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4510   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
4511 }
4512 
4513 void MacroAssembler::verify_tlab() {
4514 #ifdef ASSERT
4515   if (UseTLAB && VerifyOops) {
4516     Label next, ok;
4517 
4518     stp(rscratch2, rscratch1, Address(pre(sp, -16)));
4519 
4520     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4521     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4522     cmp(rscratch2, rscratch1);
4523     br(Assembler::HS, next);
4524     STOP("assert(top >= start)");
4525     should_not_reach_here();
4526 
4527     bind(next);
4528     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4529     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4530     cmp(rscratch2, rscratch1);
4531     br(Assembler::HS, ok);
4532     STOP("assert(top <= end)");
4533     should_not_reach_here();
4534 
4535     bind(ok);
4536     ldp(rscratch2, rscratch1, Address(post(sp, 16)));
4537   }
4538 #endif
4539 }
4540 













4541 // Writes to stack successive pages until offset reached to check for
4542 // stack overflow + shadow pages.  This clobbers tmp.
4543 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
4544   assert_different_registers(tmp, size, rscratch1);
4545   mov(tmp, sp);
4546   // Bang stack for total size given plus shadow page size.
4547   // Bang one page at a time because large size can bang beyond yellow and
4548   // red zones.
4549   Label loop;
4550   mov(rscratch1, os::vm_page_size());
4551   bind(loop);
4552   lea(tmp, Address(tmp, -os::vm_page_size()));
4553   subsw(size, size, rscratch1);
4554   str(size, Address(tmp));
4555   br(Assembler::GT, loop);
4556 
4557   // Bang down shadow pages too.
4558   // At this point, (tmp-0) is the last address touched, so don't
4559   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
4560   // was post-decremented.)  Skip this address by starting at i=1, and

4646 }
4647 
4648 void MacroAssembler::remove_frame(int framesize) {
4649   assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR");
4650   assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
4651   if (framesize < ((1 << 9) + 2 * wordSize)) {
4652     ldp(rfp, lr, Address(sp, framesize - 2 * wordSize));
4653     add(sp, sp, framesize);
4654   } else {
4655     if (framesize < ((1 << 12) + 2 * wordSize))
4656       add(sp, sp, framesize - 2 * wordSize);
4657     else {
4658       mov(rscratch1, framesize - 2 * wordSize);
4659       add(sp, sp, rscratch1);
4660     }
4661     ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
4662   }
4663   authenticate_return_address();
4664 }
4665 



















































4666 
4667 // This method counts leading positive bytes (highest bit not set) in provided byte array
4668 address MacroAssembler::count_positives(Register ary1, Register len, Register result) {
4669     // Simple and most common case of aligned small array which is not at the
4670     // end of memory page is placed here. All other cases are in stub.
4671     Label LOOP, END, STUB, STUB_LONG, SET_RESULT, DONE;
4672     const uint64_t UPPER_BIT_MASK=0x8080808080808080;
4673     assert_different_registers(ary1, len, result);
4674 
4675     mov(result, len);
4676     cmpw(len, 0);
4677     br(LE, DONE);
4678     cmpw(len, 4 * wordSize);
4679     br(GE, STUB_LONG); // size > 32 then go to stub
4680 
4681     int shift = 64 - exact_log2(os::vm_page_size());
4682     lsl(rscratch1, ary1, shift);
4683     mov(rscratch2, (size_t)(4 * wordSize) << shift);
4684     adds(rscratch2, rscratch1, rscratch2);  // At end of page?
4685     br(CS, STUB); // at the end of page then go to stub

5554 // On other systems, the helper is a usual C function.
5555 //
5556 void MacroAssembler::get_thread(Register dst) {
5557   RegSet saved_regs =
5558     LINUX_ONLY(RegSet::range(r0, r1)  + lr - dst)
5559     NOT_LINUX (RegSet::range(r0, r17) + lr - dst);
5560 
5561   protect_return_address();
5562   push(saved_regs, sp);
5563 
5564   mov(lr, CAST_FROM_FN_PTR(address, JavaThread::aarch64_get_thread_helper));
5565   blr(lr);
5566   if (dst != c_rarg0) {
5567     mov(dst, c_rarg0);
5568   }
5569 
5570   pop(saved_regs, sp);
5571   authenticate_return_address();
5572 }
5573 
















































































































































































































































































































































































































































5574 void MacroAssembler::cache_wb(Address line) {
5575   assert(line.getMode() == Address::base_plus_offset, "mode should be base_plus_offset");
5576   assert(line.index() == noreg, "index should be noreg");
5577   assert(line.offset() == 0, "offset should be 0");
5578   // would like to assert this
5579   // assert(line._ext.shift == 0, "shift should be zero");
5580   if (VM_Version::supports_dcpop()) {
5581     // writeback using clear virtual address to point of persistence
5582     dc(Assembler::CVAP, line.base());
5583   } else {
5584     // no need to generate anything as Unsafe.writebackMemory should
5585     // never invoke this stub
5586   }
5587 }
5588 
5589 void MacroAssembler::cache_wbsync(bool is_pre) {
5590   // we only need a barrier post sync
5591   if (!is_pre) {
5592     membar(Assembler::AnyAny);
5593   }

  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include <sys/types.h>
  27 
  28 #include "precompiled.hpp"
  29 #include "jvm.h"
  30 #include "asm/assembler.hpp"
  31 #include "asm/assembler.inline.hpp"
  32 #include "ci/ciEnv.hpp"
  33 #include "ci/ciInlineKlass.hpp"
  34 #include "compiler/oopMap.hpp"
  35 #include "gc/shared/barrierSet.hpp"
  36 #include "gc/shared/barrierSetAssembler.hpp"
  37 #include "gc/shared/cardTableBarrierSet.hpp"
  38 #include "gc/shared/cardTable.hpp"
  39 #include "gc/shared/collectedHeap.hpp"
  40 #include "gc/shared/tlab_globals.hpp"
  41 #include "interpreter/bytecodeHistogram.hpp"
  42 #include "interpreter/interpreter.hpp"
  43 #include "compiler/compileTask.hpp"
  44 #include "compiler/disassembler.hpp"
  45 #include "memory/resourceArea.hpp"
  46 #include "memory/universe.hpp"
  47 #include "nativeInst_aarch64.hpp"
  48 #include "oops/accessDecorators.hpp"
  49 #include "oops/compressedOops.inline.hpp"
  50 #include "oops/klass.inline.hpp"
  51 #include "runtime/continuation.hpp"
  52 #include "runtime/icache.hpp"
  53 #include "runtime/interfaceSupport.inline.hpp"
  54 #include "runtime/javaThread.hpp"
  55 #include "runtime/jniHandles.inline.hpp"
  56 #include "runtime/sharedRuntime.hpp"
  57 #include "runtime/signature_cc.hpp"
  58 #include "runtime/stubRoutines.hpp"
  59 #include "utilities/powerOfTwo.hpp"
  60 #include "vmreg_aarch64.inline.hpp"
  61 #ifdef COMPILER1
  62 #include "c1/c1_LIRAssembler.hpp"
  63 #endif
  64 #ifdef COMPILER2
  65 #include "oops/oop.hpp"
  66 #include "opto/compile.hpp"
  67 #include "opto/node.hpp"
  68 #include "opto/output.hpp"
  69 #endif
  70 
  71 #ifdef PRODUCT
  72 #define BLOCK_COMMENT(str) /* nothing */
  73 #else
  74 #define BLOCK_COMMENT(str) block_comment(str)
  75 #endif
  76 #define STOP(str) stop(str);
  77 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  78 
  79 #ifdef ASSERT
  80 extern "C" void disnm(intptr_t p);

1098   while (offset() % modulus != 0) nop();
1099 }
1100 
1101 void MacroAssembler::post_call_nop() {
1102   if (!Continuations::enabled()) {
1103     return;
1104   }
1105   InstructionMark im(this);
1106   relocate(post_call_nop_Relocation::spec());
1107   nop();
1108   movk(zr, 0);
1109   movk(zr, 0);
1110 }
1111 
1112 // these are no-ops overridden by InterpreterMacroAssembler
1113 
1114 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { }
1115 
1116 void MacroAssembler::check_and_handle_popframe(Register java_thread) { }
1117 
1118 void MacroAssembler::get_default_value_oop(Register inline_klass, Register temp_reg, Register obj) {
1119 #ifdef ASSERT
1120   {
1121     Label done_check;
1122     test_klass_is_inline_type(inline_klass, temp_reg, done_check);
1123     stop("get_default_value_oop from non inline type klass");
1124     bind(done_check);
1125   }
1126 #endif
1127   Register offset = temp_reg;
1128   // Getting the offset of the pre-allocated default value
1129   ldr(offset, Address(inline_klass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())));
1130   ldr(offset, Address(offset, in_bytes(InlineKlass::default_value_offset_offset())));
1131 
1132   // Getting the mirror
1133   ldr(obj, Address(inline_klass, in_bytes(Klass::java_mirror_offset())));
1134   resolve_oop_handle(obj, inline_klass, temp_reg);
1135 
1136   // Getting the pre-allocated default value from the mirror
1137   Address field(obj, offset);
1138   load_heap_oop(obj, field);
1139 }
1140 
1141 void MacroAssembler::get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj) {
1142 #ifdef ASSERT
1143   {
1144     Label done_check;
1145     test_klass_is_empty_inline_type(inline_klass, temp_reg, done_check);
1146     stop("get_empty_value from non-empty inline klass");
1147     bind(done_check);
1148   }
1149 #endif
1150   get_default_value_oop(inline_klass, temp_reg, obj);
1151 }
1152 
1153 // Look up the method for a megamorphic invokeinterface call.
1154 // The target method is determined by <intf_klass, itable_index>.
1155 // The receiver klass is in recv_klass.
1156 // On success, the result will be in method_result, and execution falls through.
1157 // On failure, execution transfers to the given label.
1158 void MacroAssembler::lookup_interface_method(Register recv_klass,
1159                                              Register intf_klass,
1160                                              RegisterOrConstant itable_index,
1161                                              Register method_result,
1162                                              Register scan_temp,
1163                                              Label& L_no_such_interface,
1164                          bool return_method) {
1165   assert_different_registers(recv_klass, intf_klass, scan_temp);
1166   assert_different_registers(method_result, intf_klass, scan_temp);
1167   assert(recv_klass != method_result || !return_method,
1168      "recv_klass can be destroyed when method isn't needed");
1169   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
1170          "caller must use same register for non-constant itable index as for method");
1171 
1172   // Compute start of first itableOffsetEntry (which is at the end of the vtable)

1483   ldrb(scratch, Address(klass, InstanceKlass::init_state_offset()));
1484   subs(zr, scratch, InstanceKlass::fully_initialized);
1485   br(Assembler::EQ, *L_fast_path);
1486 
1487   // Fast path check: current thread is initializer thread
1488   ldr(scratch, Address(klass, InstanceKlass::init_thread_offset()));
1489   cmp(rthread, scratch);
1490 
1491   if (L_slow_path == &L_fallthrough) {
1492     br(Assembler::EQ, *L_fast_path);
1493     bind(*L_slow_path);
1494   } else if (L_fast_path == &L_fallthrough) {
1495     br(Assembler::NE, *L_slow_path);
1496     bind(*L_fast_path);
1497   } else {
1498     Unimplemented();
1499   }
1500 }
1501 
1502 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
1503   if (!VerifyOops || VerifyAdapterSharing) {
1504     // Below address of the code string confuses VerifyAdapterSharing
1505     // because it may differ between otherwise equivalent adapters.
1506     return;
1507   }
1508 
1509   // Pass register number to verify_oop_subroutine
1510   const char* b = NULL;
1511   {
1512     ResourceMark rm;
1513     stringStream ss;
1514     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
1515     b = code_string(ss.as_string());
1516   }
1517   BLOCK_COMMENT("verify_oop {");
1518 
1519   strip_return_address(); // This might happen within a stack frame.
1520   protect_return_address();
1521   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1522   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1523 
1524   mov(r0, reg);
1525   movptr(rscratch1, (uintptr_t)(address)b);
1526 
1527   // call indirectly to solve generation ordering problem
1528   lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1529   ldr(rscratch2, Address(rscratch2));
1530   blr(rscratch2);
1531 
1532   ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1533   ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1534   authenticate_return_address();
1535 
1536   BLOCK_COMMENT("} verify_oop");
1537 }
1538 
1539 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
1540   if (!VerifyOops || VerifyAdapterSharing) {
1541     // Below address of the code string confuses VerifyAdapterSharing
1542     // because it may differ between otherwise equivalent adapters.
1543     return;
1544   }
1545 
1546   const char* b = NULL;
1547   {
1548     ResourceMark rm;
1549     stringStream ss;
1550     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
1551     b = code_string(ss.as_string());
1552   }
1553   BLOCK_COMMENT("verify_oop_addr {");
1554 
1555   strip_return_address(); // This might happen within a stack frame.
1556   protect_return_address();
1557   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1558   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1559 
1560   // addr may contain sp so we will have to adjust it based on the
1561   // pushes that we just did.
1562   if (addr.uses(sp)) {
1563     lea(r0, addr);
1564     ldr(r0, Address(r0, 4 * wordSize));

1606   stp(rscratch1, rmethod, Address(pre(sp, -2 * wordSize)));
1607 
1608   mov(rscratch1, entry_point);
1609   blr(rscratch1);
1610   if (retaddr)
1611     bind(*retaddr);
1612 
1613   ldp(rscratch1, rmethod, Address(post(sp, 2 * wordSize)));
1614 }
1615 
1616 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
1617   call_VM_leaf_base(entry_point, number_of_arguments);
1618 }
1619 
1620 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
1621   pass_arg0(this, arg_0);
1622   call_VM_leaf_base(entry_point, 1);
1623 }
1624 
1625 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1626   assert_different_registers(arg_1, c_rarg0);
1627   pass_arg0(this, arg_0);
1628   pass_arg1(this, arg_1);
1629   call_VM_leaf_base(entry_point, 2);
1630 }
1631 
1632 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
1633                                   Register arg_1, Register arg_2) {
1634   assert_different_registers(arg_1, c_rarg0);
1635   assert_different_registers(arg_2, c_rarg0, c_rarg1);
1636   pass_arg0(this, arg_0);
1637   pass_arg1(this, arg_1);
1638   pass_arg2(this, arg_2);
1639   call_VM_leaf_base(entry_point, 3);
1640 }
1641 
1642 void MacroAssembler::super_call_VM_leaf(address entry_point) {
1643   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1644 }
1645 
1646 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1647   pass_arg0(this, arg_0);
1648   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1649 }
1650 
1651 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1652 
1653   assert(arg_0 != c_rarg1, "smashed arg");
1654   pass_arg1(this, arg_1);
1655   pass_arg0(this, arg_0);
1656   MacroAssembler::call_VM_leaf_base(entry_point, 2);
1657 }
1658 
1659 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1660   assert(arg_0 != c_rarg2, "smashed arg");
1661   assert(arg_1 != c_rarg2, "smashed arg");
1662   pass_arg2(this, arg_2);
1663   assert(arg_0 != c_rarg1, "smashed arg");
1664   pass_arg1(this, arg_1);
1665   pass_arg0(this, arg_0);

1675   assert(arg_1 != c_rarg2, "smashed arg");
1676   pass_arg2(this, arg_2);
1677   assert(arg_0 != c_rarg1, "smashed arg");
1678   pass_arg1(this, arg_1);
1679   pass_arg0(this, arg_0);
1680   MacroAssembler::call_VM_leaf_base(entry_point, 4);
1681 }
1682 
1683 void MacroAssembler::null_check(Register reg, int offset) {
1684   if (needs_explicit_null_check(offset)) {
1685     // provoke OS NULL exception if reg = NULL by
1686     // accessing M[reg] w/o changing any registers
1687     // NOTE: this is plenty to provoke a segv
1688     ldr(zr, Address(reg));
1689   } else {
1690     // nothing to do, (later) access of M[reg + offset]
1691     // will provoke OS NULL exception if reg = NULL
1692   }
1693 }
1694 
1695 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
1696   assert_different_registers(markword, rscratch2);
1697   andr(markword, markword, markWord::inline_type_mask_in_place);
1698   mov(rscratch2, markWord::inline_type_pattern);
1699   cmp(markword, rscratch2);
1700   br(Assembler::EQ, is_inline_type);
1701 }
1702 
1703 void MacroAssembler::test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type) {
1704   ldrw(temp_reg, Address(klass, Klass::access_flags_offset()));
1705   andr(temp_reg, temp_reg, JVM_ACC_VALUE);
1706   cbnz(temp_reg, is_inline_type);
1707 }
1708 
1709 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type) {
1710   assert_different_registers(tmp, rscratch1);
1711   cbz(object, not_inline_type);
1712   const int is_inline_type_mask = markWord::inline_type_pattern;
1713   ldr(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
1714   mov(rscratch1, is_inline_type_mask);
1715   andr(tmp, tmp, rscratch1);
1716   cmp(tmp, rscratch1);
1717   br(Assembler::NE, not_inline_type);
1718 }
1719 
1720 void MacroAssembler::test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type) {
1721 #ifdef ASSERT
1722   {
1723     Label done_check;
1724     test_klass_is_inline_type(klass, temp_reg, done_check);
1725     stop("test_klass_is_empty_inline_type with non inline type klass");
1726     bind(done_check);
1727   }
1728 #endif
1729   ldrw(temp_reg, Address(klass, InstanceKlass::misc_flags_offset()));
1730   andr(temp_reg, temp_reg, InstanceKlass::misc_flag_is_empty_inline_type());
1731   cbnz(temp_reg, is_empty_inline_type);
1732 }
1733 
1734 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
1735   assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86
1736   tbnz(flags, ConstantPoolCacheEntry::is_null_free_inline_type_shift, is_null_free_inline_type);
1737 }
1738 
1739 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
1740   assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86
1741   tbz(flags, ConstantPoolCacheEntry::is_null_free_inline_type_shift, not_null_free_inline_type);
1742 }
1743 
1744 void MacroAssembler::test_field_is_inlined(Register flags, Register temp_reg, Label& is_flattened) {
1745   assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86
1746   tbnz(flags, ConstantPoolCacheEntry::is_inlined_shift, is_flattened);
1747 }
1748 
1749 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label) {
1750   Label test_mark_word;
1751   // load mark word
1752   ldr(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
1753   // check displaced
1754   tst(temp_reg, markWord::unlocked_value);
1755   br(Assembler::NE, test_mark_word);
1756   // slow path use klass prototype
1757   load_prototype_header(temp_reg, oop);
1758 
1759   bind(test_mark_word);
1760   andr(temp_reg, temp_reg, test_bit);
1761   if (jmp_set) {
1762     cbnz(temp_reg, jmp_label);
1763   } else {
1764     cbz(temp_reg, jmp_label);
1765   }
1766 }
1767 
1768 void MacroAssembler::test_flattened_array_oop(Register oop, Register temp_reg, Label& is_flattened_array) {
1769   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flattened_array);
1770 }
1771 
1772 void MacroAssembler::test_non_flattened_array_oop(Register oop, Register temp_reg,
1773                                                   Label&is_non_flattened_array) {
1774   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flattened_array);
1775 }
1776 
1777 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array) {
1778   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
1779 }
1780 
1781 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
1782   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
1783 }
1784 
1785 void MacroAssembler::test_flattened_array_layout(Register lh, Label& is_flattened_array) {
1786   tst(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
1787   br(Assembler::NE, is_flattened_array);
1788 }
1789 
1790 void MacroAssembler::test_non_flattened_array_layout(Register lh, Label& is_non_flattened_array) {
1791   tst(lh, Klass::_lh_array_tag_flat_value_bit_inplace);
1792   br(Assembler::EQ, is_non_flattened_array);
1793 }
1794 
1795 void MacroAssembler::test_null_free_array_layout(Register lh, Label& is_null_free_array) {
1796   tst(lh, Klass::_lh_null_free_array_bit_inplace);
1797   br(Assembler::NE, is_null_free_array);
1798 }
1799 
1800 void MacroAssembler::test_non_null_free_array_layout(Register lh, Label& is_non_null_free_array) {
1801   tst(lh, Klass::_lh_null_free_array_bit_inplace);
1802   br(Assembler::EQ, is_non_null_free_array);
1803 }
1804 
1805 // MacroAssembler protected routines needed to implement
1806 // public methods
1807 
1808 void MacroAssembler::mov(Register r, Address dest) {
1809   code_section()->relocate(pc(), dest.rspec());
1810   uint64_t imm64 = (uint64_t)dest.target();
1811   movptr(r, imm64);
1812 }
1813 
1814 // Move a constant pointer into r.  In AArch64 mode the virtual
1815 // address space is 48 bits in size, so we only need three
1816 // instructions to create a patchable instruction sequence that can
1817 // reach anywhere.
1818 void MacroAssembler::movptr(Register r, uintptr_t imm64) {
1819 #ifndef PRODUCT
1820   {
1821     char buffer[64];
1822     snprintf(buffer, sizeof(buffer), "0x%" PRIX64, (uint64_t)imm64);
1823     block_comment(buffer);
1824   }

4187   adrp(rscratch1, src2, offset);
4188   ldr(rscratch1, Address(rscratch1, offset));
4189   cmp(src1, rscratch1);
4190 }
4191 
4192 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
4193   cmp(obj1, obj2);
4194 }
4195 
4196 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
4197   load_method_holder(rresult, rmethod);
4198   ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
4199 }
4200 
4201 void MacroAssembler::load_method_holder(Register holder, Register method) {
4202   ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
4203   ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
4204   ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
4205 }
4206 
4207 void MacroAssembler::load_metadata(Register dst, Register src) {
4208   if (UseCompressedClassPointers) {
4209     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4210   } else {
4211     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4212   }
4213 }
4214 
4215 void MacroAssembler::load_klass(Register dst, Register src) {
4216   if (UseCompressedClassPointers) {
4217     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4218     decode_klass_not_null(dst);
4219   } else {
4220     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
4221   }
4222 }
4223 
4224 // ((OopHandle)result).resolve();
4225 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) {
4226   // OopHandle::resolve is an indirection.
4227   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp1, tmp2);
4228 }
4229 
4230 // ((WeakHandle)result).resolve();
4231 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2) {
4232   assert_different_registers(result, tmp1, tmp2);
4233   Label resolved;
4234 

4253 
4254 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
4255   if (UseCompressedClassPointers) {
4256     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4257     if (CompressedKlassPointers::base() == NULL) {
4258       cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
4259       return;
4260     } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
4261                && CompressedKlassPointers::shift() == 0) {
4262       // Only the bottom 32 bits matter
4263       cmpw(trial_klass, tmp);
4264       return;
4265     }
4266     decode_klass_not_null(tmp);
4267   } else {
4268     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
4269   }
4270   cmp(trial_klass, tmp);
4271 }
4272 
4273 void MacroAssembler::load_prototype_header(Register dst, Register src) {
4274   load_klass(dst, src);
4275   ldr(dst, Address(dst, Klass::prototype_header_offset()));
4276 }
4277 
4278 void MacroAssembler::store_klass(Register dst, Register src) {
4279   // FIXME: Should this be a store release?  concurrent gcs assumes
4280   // klass length is valid if klass field is not null.
4281   if (UseCompressedClassPointers) {
4282     encode_klass_not_null(src);
4283     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4284   } else {
4285     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
4286   }
4287 }
4288 
4289 void MacroAssembler::store_klass_gap(Register dst, Register src) {
4290   if (UseCompressedClassPointers) {
4291     // Store to klass gap in destination
4292     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
4293   }
4294 }
4295 
4296 // Algorithm must match CompressedOops::encode.
4297 void MacroAssembler::encode_heap_oop(Register d, Register s) {

4582   if (as_raw) {
4583     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2);
4584   } else {
4585     bs->load_at(this, decorators, type, dst, src, tmp1, tmp2);
4586   }
4587 }
4588 
4589 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
4590                                      Address dst, Register src,
4591                                      Register tmp1, Register tmp2, Register tmp3) {
4592   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4593   decorators = AccessInternal::decorator_fixup(decorators);
4594   bool as_raw = (decorators & AS_RAW) != 0;
4595   if (as_raw) {
4596     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
4597   } else {
4598     bs->store_at(this, decorators, type, dst, src, tmp1, tmp2, tmp3);
4599   }
4600 }
4601 
4602 void MacroAssembler::access_value_copy(DecoratorSet decorators, Register src, Register dst,
4603                                        Register inline_klass) {
4604   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4605   bs->value_copy(this, decorators, src, dst, inline_klass);
4606 }
4607 
4608 void MacroAssembler::first_field_offset(Register inline_klass, Register offset) {
4609   ldr(offset, Address(inline_klass, InstanceKlass::adr_inlineklass_fixed_block_offset()));
4610   ldrw(offset, Address(offset, InlineKlass::first_field_offset_offset()));
4611 }
4612 
4613 void MacroAssembler::data_for_oop(Register oop, Register data, Register inline_klass) {
4614   // ((address) (void*) o) + vk->first_field_offset();
4615   Register offset = (data == oop) ? rscratch1 : data;
4616   first_field_offset(inline_klass, offset);
4617   if (data == oop) {
4618     add(data, data, offset);
4619   } else {
4620     lea(data, Address(oop, offset));
4621   }
4622 }
4623 
4624 void MacroAssembler::data_for_value_array_index(Register array, Register array_klass,
4625                                                 Register index, Register data) {
4626   assert_different_registers(array, array_klass, index);
4627   assert_different_registers(rscratch1, array, index);
4628 
4629   // array->base() + (index << Klass::layout_helper_log2_element_size(lh));
4630   ldrw(rscratch1, Address(array_klass, Klass::layout_helper_offset()));
4631 
4632   // Klass::layout_helper_log2_element_size(lh)
4633   // (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask;
4634   lsr(rscratch1, rscratch1, Klass::_lh_log2_element_size_shift);
4635   andr(rscratch1, rscratch1, Klass::_lh_log2_element_size_mask);
4636   lslv(index, index, rscratch1);
4637 
4638   add(data, array, index);
4639   add(data, data, arrayOopDesc::base_offset_in_bytes(T_PRIMITIVE_OBJECT));
4640 }
4641 
4642 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
4643                                    Register tmp2, DecoratorSet decorators) {
4644   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
4645 }
4646 
4647 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
4648                                             Register tmp2, DecoratorSet decorators) {
4649   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, tmp2);
4650 }
4651 
4652 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
4653                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
4654   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2, tmp3);
4655 }
4656 
4657 // Used for storing NULLs.
4658 void MacroAssembler::store_heap_oop_null(Address dst) {
4659   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
4660 }
4661 

4698     oop_index = oop_recorder()->allocate_metadata_index(obj);
4699   } else {
4700     oop_index = oop_recorder()->find_index(obj);
4701   }
4702   RelocationHolder rspec = metadata_Relocation::spec(oop_index);
4703   mov(dst, Address((address)obj, rspec));
4704 }
4705 
4706 Address MacroAssembler::constant_oop_address(jobject obj) {
4707 #ifdef ASSERT
4708   {
4709     ThreadInVMfromUnknown tiv;
4710     assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
4711     assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "not an oop");
4712   }
4713 #endif
4714   int oop_index = oop_recorder()->find_index(obj);
4715   return Address((address)obj, oop_Relocation::spec(oop_index));
4716 }
4717 
4718 // Object / value buffer allocation...
4719 void MacroAssembler::allocate_instance(Register klass, Register new_obj,
4720                                        Register t1, Register t2,
4721                                        bool clear_fields, Label& alloc_failed)
4722 {
4723   Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop;
4724   Register layout_size = t1;
4725   assert(new_obj == r0, "needs to be r0");
4726   assert_different_registers(klass, new_obj, t1, t2);
4727 
4728   // get instance_size in InstanceKlass (scaled to a count of bytes)
4729   ldrw(layout_size, Address(klass, Klass::layout_helper_offset()));
4730   // test to see if it has a finalizer or is malformed in some way
4731   tst(layout_size, Klass::_lh_instance_slow_path_bit);
4732   br(Assembler::NE, slow_case_no_pop);
4733 
4734   // Allocate the instance:
4735   //  If TLAB is enabled:
4736   //    Try to allocate in the TLAB.
4737   //    If fails, go to the slow path.
4738   //    Initialize the allocation.
4739   //    Exit.
4740   //
4741   //  Go to slow path.
4742 
4743   if (UseTLAB) {
4744     push(klass);
4745     tlab_allocate(new_obj, layout_size, 0, klass, t2, slow_case);
4746     if (ZeroTLAB || (!clear_fields)) {
4747       // the fields have been already cleared
4748       b(initialize_header);
4749     } else {
4750       // initialize both the header and fields
4751       b(initialize_object);
4752     }
4753 
4754     if (clear_fields) {
4755       // The object is initialized before the header.  If the object size is
4756       // zero, go directly to the header initialization.
4757       bind(initialize_object);
4758       subs(layout_size, layout_size, sizeof(oopDesc));
4759       br(Assembler::EQ, initialize_header);
4760 
4761       // Initialize topmost object field, divide size by 8, check if odd and
4762       // test if zero.
4763 
4764   #ifdef ASSERT
4765       // make sure instance_size was multiple of 8
4766       Label L;
4767       tst(layout_size, 7);
4768       br(Assembler::EQ, L);
4769       stop("object size is not multiple of 8 - adjust this code");
4770       bind(L);
4771       // must be > 0, no extra check needed here
4772   #endif
4773 
4774       lsr(layout_size, layout_size, LogBytesPerLong);
4775 
4776       // initialize remaining object fields: instance_size was a multiple of 8
4777       {
4778         Label loop;
4779         Register base = t2;
4780 
4781         bind(loop);
4782         add(rscratch1, new_obj, layout_size, Assembler::LSL, LogBytesPerLong);
4783         str(zr, Address(rscratch1, sizeof(oopDesc) - 1*oopSize));
4784         subs(layout_size, layout_size, 1);
4785         br(Assembler::NE, loop);
4786       }
4787     } // clear_fields
4788 
4789     // initialize object header only.
4790     bind(initialize_header);
4791     pop(klass);
4792     Register mark_word = t2;
4793     ldr(mark_word, Address(klass, Klass::prototype_header_offset()));
4794     str(mark_word, Address(new_obj, oopDesc::mark_offset_in_bytes ()));
4795     store_klass_gap(new_obj, zr);  // zero klass gap for compressed oops
4796     mov(t2, klass);         // preserve klass
4797     store_klass(new_obj, t2);  // src klass reg is potentially compressed
4798 
4799     // TODO: Valhalla removed SharedRuntime::dtrace_object_alloc from here ?
4800 
4801     b(done);
4802   }
4803 
4804   if (UseTLAB) {
4805     bind(slow_case);
4806     pop(klass);
4807   }
4808   bind(slow_case_no_pop);
4809   b(alloc_failed);
4810 
4811   bind(done);
4812 }
4813 
4814 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4815 void MacroAssembler::tlab_allocate(Register obj,
4816                                    Register var_size_in_bytes,
4817                                    int con_size_in_bytes,
4818                                    Register t1,
4819                                    Register t2,
4820                                    Label& slow_case) {
4821   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4822   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
4823 }
4824 
4825 void MacroAssembler::verify_tlab() {
4826 #ifdef ASSERT
4827   if (UseTLAB && VerifyOops) {
4828     Label next, ok;
4829 
4830     stp(rscratch2, rscratch1, Address(pre(sp, -16)));
4831 
4832     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4833     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4834     cmp(rscratch2, rscratch1);
4835     br(Assembler::HS, next);
4836     STOP("assert(top >= start)");
4837     should_not_reach_here();
4838 
4839     bind(next);
4840     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4841     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4842     cmp(rscratch2, rscratch1);
4843     br(Assembler::HS, ok);
4844     STOP("assert(top <= end)");
4845     should_not_reach_here();
4846 
4847     bind(ok);
4848     ldp(rscratch2, rscratch1, Address(post(sp, 16)));
4849   }
4850 #endif
4851 }
4852 
4853 void MacroAssembler::get_inline_type_field_klass(Register klass, Register index, Register inline_klass) {
4854   ldr(inline_klass, Address(klass, InstanceKlass::inline_type_field_klasses_offset()));
4855 #ifdef ASSERT
4856   {
4857     Label done;
4858     cbnz(inline_klass, done);
4859     stop("get_inline_type_field_klass contains no inline klass");
4860     bind(done);
4861   }
4862 #endif
4863   ldr(inline_klass, Address(inline_klass, index, Address::lsl(3)));
4864 }
4865 
4866 // Writes to stack successive pages until offset reached to check for
4867 // stack overflow + shadow pages.  This clobbers tmp.
4868 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
4869   assert_different_registers(tmp, size, rscratch1);
4870   mov(tmp, sp);
4871   // Bang stack for total size given plus shadow page size.
4872   // Bang one page at a time because large size can bang beyond yellow and
4873   // red zones.
4874   Label loop;
4875   mov(rscratch1, os::vm_page_size());
4876   bind(loop);
4877   lea(tmp, Address(tmp, -os::vm_page_size()));
4878   subsw(size, size, rscratch1);
4879   str(size, Address(tmp));
4880   br(Assembler::GT, loop);
4881 
4882   // Bang down shadow pages too.
4883   // At this point, (tmp-0) is the last address touched, so don't
4884   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
4885   // was post-decremented.)  Skip this address by starting at i=1, and

4971 }
4972 
4973 void MacroAssembler::remove_frame(int framesize) {
4974   assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR");
4975   assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
4976   if (framesize < ((1 << 9) + 2 * wordSize)) {
4977     ldp(rfp, lr, Address(sp, framesize - 2 * wordSize));
4978     add(sp, sp, framesize);
4979   } else {
4980     if (framesize < ((1 << 12) + 2 * wordSize))
4981       add(sp, sp, framesize - 2 * wordSize);
4982     else {
4983       mov(rscratch1, framesize - 2 * wordSize);
4984       add(sp, sp, rscratch1);
4985     }
4986     ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
4987   }
4988   authenticate_return_address();
4989 }
4990 
4991 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
4992   if (needs_stack_repair) {
4993     // Remove the extension of the caller's frame used for inline type unpacking
4994     //
4995     // Right now the stack looks like this:
4996     //
4997     // | Arguments from caller     |
4998     // |---------------------------|  <-- caller's SP
4999     // | Saved LR #1               |
5000     // | Saved FP #1               |
5001     // |---------------------------|
5002     // | Extension space for       |
5003     // |   inline arg (un)packing  |
5004     // |---------------------------|  <-- start of this method's frame
5005     // | Saved LR #2               |
5006     // | Saved FP #2               |
5007     // |---------------------------|  <-- FP
5008     // | sp_inc                    |
5009     // | method locals             |
5010     // |---------------------------|  <-- SP
5011     //
5012     // There are two copies of FP and LR on the stack. They will be identical
5013     // unless the caller has been deoptimized, in which case LR #1 will be patched
5014     // to point at the deopt blob, and LR #2 will still point into the old method.
5015     //
5016     // The sp_inc stack slot holds the total size of the frame including the
5017     // extension space minus two words for the saved FP and LR.
5018 
5019     int sp_inc_offset = initial_framesize - 3 * wordSize;  // Immediately below saved LR and FP
5020 
5021     ldr(rscratch1, Address(sp, sp_inc_offset));
5022     add(sp, sp, rscratch1);
5023     ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
5024   } else {
5025     remove_frame(initial_framesize);
5026   }
5027 }
5028 
5029 void MacroAssembler::save_stack_increment(int sp_inc, int frame_size) {
5030   int real_frame_size = frame_size + sp_inc;
5031   assert(sp_inc == 0 || sp_inc > 2*wordSize, "invalid sp_inc value");
5032   assert(real_frame_size >= 2*wordSize, "frame size must include FP/LR space");
5033   assert((real_frame_size & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
5034 
5035   int sp_inc_offset = frame_size - 3 * wordSize;  // Immediately below saved LR and FP
5036 
5037   // Subtract two words for the saved FP and LR as these will be popped
5038   // separately. See remove_frame above.
5039   mov(rscratch1, real_frame_size - 2*wordSize);
5040   str(rscratch1, Address(sp, sp_inc_offset));
5041 }
5042 
5043 // This method counts leading positive bytes (highest bit not set) in provided byte array
5044 address MacroAssembler::count_positives(Register ary1, Register len, Register result) {
5045     // Simple and most common case of aligned small array which is not at the
5046     // end of memory page is placed here. All other cases are in stub.
5047     Label LOOP, END, STUB, STUB_LONG, SET_RESULT, DONE;
5048     const uint64_t UPPER_BIT_MASK=0x8080808080808080;
5049     assert_different_registers(ary1, len, result);
5050 
5051     mov(result, len);
5052     cmpw(len, 0);
5053     br(LE, DONE);
5054     cmpw(len, 4 * wordSize);
5055     br(GE, STUB_LONG); // size > 32 then go to stub
5056 
5057     int shift = 64 - exact_log2(os::vm_page_size());
5058     lsl(rscratch1, ary1, shift);
5059     mov(rscratch2, (size_t)(4 * wordSize) << shift);
5060     adds(rscratch2, rscratch1, rscratch2);  // At end of page?
5061     br(CS, STUB); // at the end of page then go to stub

5930 // On other systems, the helper is a usual C function.
5931 //
5932 void MacroAssembler::get_thread(Register dst) {
5933   RegSet saved_regs =
5934     LINUX_ONLY(RegSet::range(r0, r1)  + lr - dst)
5935     NOT_LINUX (RegSet::range(r0, r17) + lr - dst);
5936 
5937   protect_return_address();
5938   push(saved_regs, sp);
5939 
5940   mov(lr, CAST_FROM_FN_PTR(address, JavaThread::aarch64_get_thread_helper));
5941   blr(lr);
5942   if (dst != c_rarg0) {
5943     mov(dst, c_rarg0);
5944   }
5945 
5946   pop(saved_regs, sp);
5947   authenticate_return_address();
5948 }
5949 
5950 #ifdef COMPILER2
5951 // C2 compiled method's prolog code
5952 // Moved here from aarch64.ad to support Valhalla code belows
5953 void MacroAssembler::verified_entry(Compile* C, int sp_inc) {
5954 
5955   // n.b. frame size includes space for return pc and rfp
5956   const long framesize = C->output()->frame_size_in_bytes();
5957 
5958   // insert a nop at the start of the prolog so we can patch in a
5959   // branch if we need to invalidate the method later
5960   nop();
5961 
5962   int bangsize = C->output()->bang_size_in_bytes();
5963   if (C->output()->need_stack_bang(bangsize))
5964     generate_stack_overflow_check(bangsize);
5965 
5966   build_frame(framesize);
5967 
5968   if (C->needs_stack_repair()) {
5969     save_stack_increment(sp_inc, framesize);
5970   }
5971 
5972   if (VerifyStackAtCalls) {
5973     Unimplemented();
5974   }
5975 }
5976 #endif // COMPILER2
5977 
5978 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
5979   assert(InlineTypeReturnedAsFields, "Inline types should never be returned as fields");
5980   // An inline type might be returned. If fields are in registers we
5981   // need to allocate an inline type instance and initialize it with
5982   // the value of the fields.
5983   Label skip;
5984   // We only need a new buffered inline type if a new one is not returned
5985   tbz(r0, 0, skip);
5986   int call_offset = -1;
5987 
5988   // Be careful not to clobber r1-7 which hold returned fields
5989   // Also do not use callee-saved registers as these may be live in the interpreter
5990   Register tmp1 = r13, tmp2 = r14, klass = r15, r0_preserved = r12;
5991 
5992   // The following code is similar to allocate_instance but has some slight differences,
5993   // e.g. object size is always not zero, sometimes it's constant; storing klass ptr after
5994   // allocating is not necessary if vk != NULL, etc. allocate_instance is not aware of these.
5995   Label slow_case;
5996   // 1. Try to allocate a new buffered inline instance either from TLAB or eden space
5997   mov(r0_preserved, r0); // save r0 for slow_case since *_allocate may corrupt it when allocation failed
5998 
5999   if (vk != NULL) {
6000     // Called from C1, where the return type is statically known.
6001     movptr(klass, (intptr_t)vk->get_InlineKlass());
6002     jint obj_size = vk->layout_helper();
6003     assert(obj_size != Klass::_lh_neutral_value, "inline class in return type must have been resolved");
6004     if (UseTLAB) {
6005       tlab_allocate(r0, noreg, obj_size, tmp1, tmp2, slow_case);
6006     } else {
6007       b(slow_case);
6008     }
6009   } else {
6010     // Call from interpreter. R0 contains ((the InlineKlass* of the return type) | 0x01)
6011     andr(klass, r0, -2);
6012     ldrw(tmp2, Address(klass, Klass::layout_helper_offset()));
6013     if (UseTLAB) {
6014       tlab_allocate(r0, tmp2, 0, tmp1, tmp2, slow_case);
6015     } else {
6016       b(slow_case);
6017     }
6018   }
6019   if (UseTLAB) {
6020     // 2. Initialize buffered inline instance header
6021     Register buffer_obj = r0;
6022     mov(rscratch1, (intptr_t)markWord::inline_type_prototype().value());
6023     str(rscratch1, Address(buffer_obj, oopDesc::mark_offset_in_bytes()));
6024     store_klass_gap(buffer_obj, zr);
6025     if (vk == NULL) {
6026       // store_klass corrupts klass, so save it for later use (interpreter case only).
6027       mov(tmp1, klass);
6028     }
6029     store_klass(buffer_obj, klass);
6030     // 3. Initialize its fields with an inline class specific handler
6031     if (vk != NULL) {
6032       far_call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint.
6033     } else {
6034       // tmp1 holds klass preserved above
6035       ldr(tmp1, Address(tmp1, InstanceKlass::adr_inlineklass_fixed_block_offset()));
6036       ldr(tmp1, Address(tmp1, InlineKlass::pack_handler_offset()));
6037       blr(tmp1);
6038     }
6039 
6040     membar(Assembler::StoreStore);
6041     b(skip);
6042   } else {
6043     // Must have already branched to slow_case above.
6044     DEBUG_ONLY(should_not_reach_here());
6045   }
6046   bind(slow_case);
6047   // We failed to allocate a new inline type, fall back to a runtime
6048   // call. Some oop field may be live in some registers but we can't
6049   // tell. That runtime call will take care of preserving them
6050   // across a GC if there's one.
6051   mov(r0, r0_preserved);
6052 
6053   if (from_interpreter) {
6054     super_call_VM_leaf(StubRoutines::store_inline_type_fields_to_buf());
6055   } else {
6056     far_call(RuntimeAddress(StubRoutines::store_inline_type_fields_to_buf()));
6057     call_offset = offset();
6058   }
6059   membar(Assembler::StoreStore);
6060 
6061   bind(skip);
6062   return call_offset;
6063 }
6064 
6065 // Move a value between registers/stack slots and update the reg_state
6066 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
6067   assert(from->is_valid() && to->is_valid(), "source and destination must be valid");
6068   if (reg_state[to->value()] == reg_written) {
6069     return true; // Already written
6070   }
6071 
6072   if (from != to && bt != T_VOID) {
6073     if (reg_state[to->value()] == reg_readonly) {
6074       return false; // Not yet writable
6075     }
6076     if (from->is_reg()) {
6077       if (to->is_reg()) {
6078         if (from->is_Register() && to->is_Register()) {
6079           mov(to->as_Register(), from->as_Register());
6080         } else if (from->is_FloatRegister() && to->is_FloatRegister()) {
6081           fmovd(to->as_FloatRegister(), from->as_FloatRegister());
6082         } else {
6083           ShouldNotReachHere();
6084         }
6085       } else {
6086         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size;
6087         Address to_addr = Address(sp, st_off);
6088         if (from->is_FloatRegister()) {
6089           if (bt == T_DOUBLE) {
6090              strd(from->as_FloatRegister(), to_addr);
6091           } else {
6092              assert(bt == T_FLOAT, "must be float");
6093              strs(from->as_FloatRegister(), to_addr);
6094           }
6095         } else {
6096           str(from->as_Register(), to_addr);
6097         }
6098       }
6099     } else {
6100       Address from_addr = Address(sp, from->reg2stack() * VMRegImpl::stack_slot_size);
6101       if (to->is_reg()) {
6102         if (to->is_FloatRegister()) {
6103           if (bt == T_DOUBLE) {
6104             ldrd(to->as_FloatRegister(), from_addr);
6105           } else {
6106             assert(bt == T_FLOAT, "must be float");
6107             ldrs(to->as_FloatRegister(), from_addr);
6108           }
6109         } else {
6110           ldr(to->as_Register(), from_addr);
6111         }
6112       } else {
6113         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size;
6114         ldr(rscratch1, from_addr);
6115         str(rscratch1, Address(sp, st_off));
6116       }
6117     }
6118   }
6119 
6120   // Update register states
6121   reg_state[from->value()] = reg_writable;
6122   reg_state[to->value()] = reg_written;
6123   return true;
6124 }
6125 
6126 // Calculate the extra stack space required for packing or unpacking inline
6127 // args and adjust the stack pointer
6128 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
6129   int sp_inc = args_on_stack * VMRegImpl::stack_slot_size;
6130   sp_inc = align_up(sp_inc, StackAlignmentInBytes);
6131   assert(sp_inc > 0, "sanity");
6132 
6133   // Save a copy of the FP and LR here for deoptimization patching and frame walking
6134   stp(rfp, lr, Address(pre(sp, -2 * wordSize)));
6135 
6136   // Adjust the stack pointer. This will be repaired on return by MacroAssembler::remove_frame
6137   if (sp_inc < (1 << 9)) {
6138     sub(sp, sp, sp_inc);   // Fits in an immediate
6139   } else {
6140     mov(rscratch1, sp_inc);
6141     sub(sp, sp, rscratch1);
6142   }
6143 
6144   return sp_inc + 2 * wordSize;  // Account for the FP/LR space
6145 }
6146 
6147 // Read all fields from an inline type oop and store the values in registers/stack slots
6148 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
6149                                           VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
6150                                           RegState reg_state[]) {
6151   assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter");
6152   assert(from->is_valid(), "source must be valid");
6153   bool progress = false;
6154 #ifdef ASSERT
6155   const int start_offset = offset();
6156 #endif
6157 
6158   Label L_null, L_notNull;
6159   // Don't use r14 as tmp because it's used for spilling (see MacroAssembler::spill_reg_for)
6160   Register tmp1 = r10;
6161   Register tmp2 = r11;
6162   Register fromReg = noreg;
6163   ScalarizedInlineArgsStream stream(sig, sig_index, to, to_count, to_index, -1);
6164   bool done = true;
6165   bool mark_done = true;
6166   VMReg toReg;
6167   BasicType bt;
6168   // Check if argument requires a null check
6169   bool null_check = false;
6170   VMReg nullCheckReg;
6171   while (stream.next(nullCheckReg, bt)) {
6172     if (sig->at(stream.sig_index())._offset == -1) {
6173       null_check = true;
6174       break;
6175     }
6176   }
6177   stream.reset(sig_index, to_index);
6178   while (stream.next(toReg, bt)) {
6179     assert(toReg->is_valid(), "destination must be valid");
6180     int idx = (int)toReg->value();
6181     if (reg_state[idx] == reg_readonly) {
6182       if (idx != from->value()) {
6183         mark_done = false;
6184       }
6185       done = false;
6186       continue;
6187     } else if (reg_state[idx] == reg_written) {
6188       continue;
6189     }
6190     assert(reg_state[idx] == reg_writable, "must be writable");
6191     reg_state[idx] = reg_written;
6192     progress = true;
6193 
6194     if (fromReg == noreg) {
6195       if (from->is_reg()) {
6196         fromReg = from->as_Register();
6197       } else {
6198         int st_off = from->reg2stack() * VMRegImpl::stack_slot_size;
6199         ldr(tmp1, Address(sp, st_off));
6200         fromReg = tmp1;
6201       }
6202       if (null_check) {
6203         // Nullable inline type argument, emit null check
6204         cbz(fromReg, L_null);
6205       }
6206     }
6207     int off = sig->at(stream.sig_index())._offset;
6208     if (off == -1) {
6209       assert(null_check, "Missing null check at");
6210       if (toReg->is_stack()) {
6211         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size;
6212         mov(tmp2, 1);
6213         str(tmp2, Address(sp, st_off));
6214       } else {
6215         mov(toReg->as_Register(), 1);
6216       }
6217       continue;
6218     }
6219     assert(off > 0, "offset in object should be positive");
6220     Address fromAddr = Address(fromReg, off);
6221     if (!toReg->is_FloatRegister()) {
6222       Register dst = toReg->is_stack() ? tmp2 : toReg->as_Register();
6223       if (is_reference_type(bt)) {
6224         load_heap_oop(dst, fromAddr);
6225       } else {
6226         bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN);
6227         load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed);
6228       }
6229       if (toReg->is_stack()) {
6230         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size;
6231         str(dst, Address(sp, st_off));
6232       }
6233     } else if (bt == T_DOUBLE) {
6234       ldrd(toReg->as_FloatRegister(), fromAddr);
6235     } else {
6236       assert(bt == T_FLOAT, "must be float");
6237       ldrs(toReg->as_FloatRegister(), fromAddr);
6238     }
6239   }
6240   if (progress && null_check) {
6241     if (done) {
6242       b(L_notNull);
6243       bind(L_null);
6244       // Set IsInit field to zero to signal that the argument is null.
6245       // Also set all oop fields to zero to make the GC happy.
6246       stream.reset(sig_index, to_index);
6247       while (stream.next(toReg, bt)) {
6248         if (sig->at(stream.sig_index())._offset == -1 ||
6249             bt == T_OBJECT || bt == T_ARRAY) {
6250           if (toReg->is_stack()) {
6251             int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size;
6252             str(zr, Address(sp, st_off));
6253           } else {
6254             mov(toReg->as_Register(), zr);
6255           }
6256         }
6257       }
6258       bind(L_notNull);
6259     } else {
6260       bind(L_null);
6261     }
6262   }
6263 
6264   sig_index = stream.sig_index();
6265   to_index = stream.regs_index();
6266 
6267   if (mark_done && reg_state[from->value()] != reg_written) {
6268     // This is okay because no one else will write to that slot
6269     reg_state[from->value()] = reg_writable;
6270   }
6271   from_index--;
6272   assert(progress || (start_offset == offset()), "should not emit code");
6273   return done;
6274 }
6275 
6276 // Pack fields back into an inline type oop
6277 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
6278                                         VMRegPair* from, int from_count, int& from_index, VMReg to,
6279                                         RegState reg_state[], Register val_array) {
6280   assert(sig->at(sig_index)._bt == T_PRIMITIVE_OBJECT, "should be at end delimiter");
6281   assert(to->is_valid(), "destination must be valid");
6282 
6283   if (reg_state[to->value()] == reg_written) {
6284     skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
6285     return true; // Already written
6286   }
6287 
6288   // The GC barrier expanded by store_heap_oop below may call into the
6289   // runtime so use callee-saved registers for any values that need to be
6290   // preserved. The GC barrier assembler should take care of saving the
6291   // Java argument registers.
6292   // TODO 8284443 Isn't it an issue if below code uses r14 as tmp when it contains a spilled value?
6293   // Be careful with r14 because it's used for spilling (see MacroAssembler::spill_reg_for).
6294   Register val_obj_tmp = r21;
6295   Register from_reg_tmp = r22;
6296   Register tmp1 = r14;
6297   Register tmp2 = r13;
6298   Register tmp3 = r12;
6299   Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register();
6300 
6301   assert_different_registers(val_obj_tmp, from_reg_tmp, tmp1, tmp2, tmp3, val_array);
6302 
6303   if (reg_state[to->value()] == reg_readonly) {
6304     if (!is_reg_in_unpacked_fields(sig, sig_index, to, from, from_count, from_index)) {
6305       skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
6306       return false; // Not yet writable
6307     }
6308     val_obj = val_obj_tmp;
6309   }
6310 
6311   int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_PRIMITIVE_OBJECT);
6312   load_heap_oop(val_obj, Address(val_array, index));
6313 
6314   ScalarizedInlineArgsStream stream(sig, sig_index, from, from_count, from_index);
6315   VMReg fromReg;
6316   BasicType bt;
6317   Label L_null;
6318   while (stream.next(fromReg, bt)) {
6319     assert(fromReg->is_valid(), "source must be valid");
6320     reg_state[fromReg->value()] = reg_writable;
6321 
6322     int off = sig->at(stream.sig_index())._offset;
6323     if (off == -1) {
6324       // Nullable inline type argument, emit null check
6325       Label L_notNull;
6326       if (fromReg->is_stack()) {
6327         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size;
6328         ldr(tmp2, Address(sp, ld_off));
6329         cbnz(tmp2, L_notNull);
6330       } else {
6331         cbnz(fromReg->as_Register(), L_notNull);
6332       }
6333       mov(val_obj, 0);
6334       b(L_null);
6335       bind(L_notNull);
6336       continue;
6337     }
6338 
6339     assert(off > 0, "offset in object should be positive");
6340     size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
6341 
6342     // Pack the scalarized field into the value object.
6343     Address dst(val_obj, off);
6344 
6345     if (!fromReg->is_FloatRegister()) {
6346       Register src;
6347       if (fromReg->is_stack()) {
6348         src = from_reg_tmp;
6349         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size;
6350         load_sized_value(src, Address(sp, ld_off), size_in_bytes, /* is_signed */ false);
6351       } else {
6352         src = fromReg->as_Register();
6353       }
6354       assert_different_registers(dst.base(), src, tmp1, tmp2, tmp3, val_array);
6355       if (is_reference_type(bt)) {
6356         store_heap_oop(dst, src, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
6357       } else {
6358         store_sized_value(dst, src, size_in_bytes);
6359       }
6360     } else if (bt == T_DOUBLE) {
6361       strd(fromReg->as_FloatRegister(), dst);
6362     } else {
6363       assert(bt == T_FLOAT, "must be float");
6364       strs(fromReg->as_FloatRegister(), dst);
6365     }
6366   }
6367   bind(L_null);
6368   sig_index = stream.sig_index();
6369   from_index = stream.regs_index();
6370 
6371   assert(reg_state[to->value()] == reg_writable, "must have already been read");
6372   bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state);
6373   assert(success, "to register must be writeable");
6374 
6375   return true;
6376 }
6377 
6378 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
6379   return (reg->is_FloatRegister()) ? v0->as_VMReg() : r14->as_VMReg();
6380 }
6381 
6382 void MacroAssembler::cache_wb(Address line) {
6383   assert(line.getMode() == Address::base_plus_offset, "mode should be base_plus_offset");
6384   assert(line.index() == noreg, "index should be noreg");
6385   assert(line.offset() == 0, "offset should be 0");
6386   // would like to assert this
6387   // assert(line._ext.shift == 0, "shift should be zero");
6388   if (VM_Version::supports_dcpop()) {
6389     // writeback using clear virtual address to point of persistence
6390     dc(Assembler::CVAP, line.base());
6391   } else {
6392     // no need to generate anything as Unsafe.writebackMemory should
6393     // never invoke this stub
6394   }
6395 }
6396 
6397 void MacroAssembler::cache_wbsync(bool is_pre) {
6398   // we only need a barrier post sync
6399   if (!is_pre) {
6400     membar(Assembler::AnyAny);
6401   }
< prev index next >