< prev index next >

src/hotspot/cpu/aarch64/macroAssembler_aarch64.cpp

Print this page

  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include <sys/types.h>
  27 
  28 #include "precompiled.hpp"
  29 #include "jvm.h"
  30 #include "asm/assembler.hpp"
  31 #include "asm/assembler.inline.hpp"

  32 #include "gc/shared/barrierSet.hpp"
  33 #include "gc/shared/barrierSetAssembler.hpp"
  34 #include "gc/shared/cardTableBarrierSet.hpp"
  35 #include "gc/shared/cardTable.hpp"
  36 #include "gc/shared/collectedHeap.hpp"
  37 #include "gc/shared/tlab_globals.hpp"
  38 #include "interpreter/bytecodeHistogram.hpp"
  39 #include "interpreter/interpreter.hpp"
  40 #include "compiler/compileTask.hpp"
  41 #include "compiler/disassembler.hpp"
  42 #include "memory/resourceArea.hpp"
  43 #include "memory/universe.hpp"
  44 #include "nativeInst_aarch64.hpp"
  45 #include "oops/accessDecorators.hpp"
  46 #include "oops/compressedOops.inline.hpp"
  47 #include "oops/klass.inline.hpp"
  48 #include "runtime/icache.hpp"
  49 #include "runtime/interfaceSupport.inline.hpp"
  50 #include "runtime/jniHandles.inline.hpp"
  51 #include "runtime/sharedRuntime.hpp"

  52 #include "runtime/stubRoutines.hpp"
  53 #include "runtime/thread.hpp"
  54 #include "utilities/powerOfTwo.hpp"

  55 #ifdef COMPILER1
  56 #include "c1/c1_LIRAssembler.hpp"
  57 #endif
  58 #ifdef COMPILER2
  59 #include "oops/oop.hpp"
  60 #include "opto/compile.hpp"
  61 #include "opto/node.hpp"
  62 #include "opto/output.hpp"
  63 #endif
  64 
  65 #ifdef PRODUCT
  66 #define BLOCK_COMMENT(str) /* nothing */
  67 #else
  68 #define BLOCK_COMMENT(str) block_comment(str)
  69 #endif
  70 #define STOP(str) stop(str);
  71 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  72 
  73 // Patch any kind of instruction; there may be several instructions.
  74 // Return the total length (in bytes) of the instructions.

 754   ldr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
 755   str(zr, Address(java_thread, JavaThread::vm_result_offset()));
 756   verify_oop(oop_result, "broken oop in call_VM_base");
 757 }
 758 
 759 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
 760   ldr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
 761   str(zr, Address(java_thread, JavaThread::vm_result_2_offset()));
 762 }
 763 
 764 void MacroAssembler::align(int modulus) {
 765   while (offset() % modulus != 0) nop();
 766 }
 767 
 768 // these are no-ops overridden by InterpreterMacroAssembler
 769 
 770 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { }
 771 
 772 void MacroAssembler::check_and_handle_popframe(Register java_thread) { }
 773 



































 774 // Look up the method for a megamorphic invokeinterface call.
 775 // The target method is determined by <intf_klass, itable_index>.
 776 // The receiver klass is in recv_klass.
 777 // On success, the result will be in method_result, and execution falls through.
 778 // On failure, execution transfers to the given label.
 779 void MacroAssembler::lookup_interface_method(Register recv_klass,
 780                                              Register intf_klass,
 781                                              RegisterOrConstant itable_index,
 782                                              Register method_result,
 783                                              Register scan_temp,
 784                                              Label& L_no_such_interface,
 785                          bool return_method) {
 786   assert_different_registers(recv_klass, intf_klass, scan_temp);
 787   assert_different_registers(method_result, intf_klass, scan_temp);
 788   assert(recv_klass != method_result || !return_method,
 789      "recv_klass can be destroyed when method isn't needed");
 790   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 791          "caller must use same register for non-constant itable index as for method");
 792 
 793   // Compute start of first itableOffsetEntry (which is at the end of the vtable)

1104   ldrb(scratch, Address(klass, InstanceKlass::init_state_offset()));
1105   subs(zr, scratch, InstanceKlass::fully_initialized);
1106   br(Assembler::EQ, *L_fast_path);
1107 
1108   // Fast path check: current thread is initializer thread
1109   ldr(scratch, Address(klass, InstanceKlass::init_thread_offset()));
1110   cmp(rthread, scratch);
1111 
1112   if (L_slow_path == &L_fallthrough) {
1113     br(Assembler::EQ, *L_fast_path);
1114     bind(*L_slow_path);
1115   } else if (L_fast_path == &L_fallthrough) {
1116     br(Assembler::NE, *L_slow_path);
1117     bind(*L_fast_path);
1118   } else {
1119     Unimplemented();
1120   }
1121 }
1122 
1123 void MacroAssembler::verify_oop(Register reg, const char* s) {
1124   if (!VerifyOops) return;




1125 
1126   // Pass register number to verify_oop_subroutine
1127   const char* b = NULL;
1128   {
1129     ResourceMark rm;
1130     stringStream ss;
1131     ss.print("verify_oop: %s: %s", reg->name(), s);
1132     b = code_string(ss.as_string());
1133   }
1134   BLOCK_COMMENT("verify_oop {");
1135 
1136   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1137   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1138 
1139   mov(r0, reg);
1140   movptr(rscratch1, (uintptr_t)(address)b);
1141 
1142   // call indirectly to solve generation ordering problem
1143   lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1144   ldr(rscratch2, Address(rscratch2));
1145   blr(rscratch2);
1146 
1147   ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1148   ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1149 
1150   BLOCK_COMMENT("} verify_oop");
1151 }
1152 
1153 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
1154   if (!VerifyOops) return;




1155 
1156   const char* b = NULL;
1157   {
1158     ResourceMark rm;
1159     stringStream ss;
1160     ss.print("verify_oop_addr: %s", s);
1161     b = code_string(ss.as_string());
1162   }
1163   BLOCK_COMMENT("verify_oop_addr {");
1164 
1165   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1166   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1167 
1168   // addr may contain sp so we will have to adjust it based on the
1169   // pushes that we just did.
1170   if (addr.uses(sp)) {
1171     lea(r0, addr);
1172     ldr(r0, Address(r0, 4 * wordSize));
1173   } else {
1174     ldr(r0, addr);

1213   stp(rscratch1, rmethod, Address(pre(sp, -2 * wordSize)));
1214 
1215   mov(rscratch1, entry_point);
1216   blr(rscratch1);
1217   if (retaddr)
1218     bind(*retaddr);
1219 
1220   ldp(rscratch1, rmethod, Address(post(sp, 2 * wordSize)));
1221 }
1222 
1223 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
1224   call_VM_leaf_base(entry_point, number_of_arguments);
1225 }
1226 
1227 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
1228   pass_arg0(this, arg_0);
1229   call_VM_leaf_base(entry_point, 1);
1230 }
1231 
1232 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {

1233   pass_arg0(this, arg_0);
1234   pass_arg1(this, arg_1);
1235   call_VM_leaf_base(entry_point, 2);
1236 }
1237 
1238 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
1239                                   Register arg_1, Register arg_2) {


1240   pass_arg0(this, arg_0);
1241   pass_arg1(this, arg_1);
1242   pass_arg2(this, arg_2);
1243   call_VM_leaf_base(entry_point, 3);
1244 }
1245 




1246 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1247   pass_arg0(this, arg_0);
1248   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1249 }
1250 
1251 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1252 
1253   assert(arg_0 != c_rarg1, "smashed arg");
1254   pass_arg1(this, arg_1);
1255   pass_arg0(this, arg_0);
1256   MacroAssembler::call_VM_leaf_base(entry_point, 2);
1257 }
1258 
1259 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1260   assert(arg_0 != c_rarg2, "smashed arg");
1261   assert(arg_1 != c_rarg2, "smashed arg");
1262   pass_arg2(this, arg_2);
1263   assert(arg_0 != c_rarg1, "smashed arg");
1264   pass_arg1(this, arg_1);
1265   pass_arg0(this, arg_0);

1275   assert(arg_1 != c_rarg2, "smashed arg");
1276   pass_arg2(this, arg_2);
1277   assert(arg_0 != c_rarg1, "smashed arg");
1278   pass_arg1(this, arg_1);
1279   pass_arg0(this, arg_0);
1280   MacroAssembler::call_VM_leaf_base(entry_point, 4);
1281 }
1282 
1283 void MacroAssembler::null_check(Register reg, int offset) {
1284   if (needs_explicit_null_check(offset)) {
1285     // provoke OS NULL exception if reg = NULL by
1286     // accessing M[reg] w/o changing any registers
1287     // NOTE: this is plenty to provoke a segv
1288     ldr(zr, Address(reg));
1289   } else {
1290     // nothing to do, (later) access of M[reg + offset]
1291     // will provoke OS NULL exception if reg = NULL
1292   }
1293 }
1294 













































































































1295 // MacroAssembler protected routines needed to implement
1296 // public methods
1297 
1298 void MacroAssembler::mov(Register r, Address dest) {
1299   code_section()->relocate(pc(), dest.rspec());
1300   uint64_t imm64 = (uint64_t)dest.target();
1301   movptr(r, imm64);
1302 }
1303 
1304 // Move a constant pointer into r.  In AArch64 mode the virtual
1305 // address space is 48 bits in size, so we only need three
1306 // instructions to create a patchable instruction sequence that can
1307 // reach anywhere.
1308 void MacroAssembler::movptr(Register r, uintptr_t imm64) {
1309 #ifndef PRODUCT
1310   {
1311     char buffer[64];
1312     snprintf(buffer, sizeof(buffer), "0x%" PRIX64, (uint64_t)imm64);
1313     block_comment(buffer);
1314   }

3589   adrp(rscratch1, src2, offset);
3590   ldr(rscratch1, Address(rscratch1, offset));
3591   cmp(src1, rscratch1);
3592 }
3593 
3594 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3595   cmp(obj1, obj2);
3596 }
3597 
3598 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3599   load_method_holder(rresult, rmethod);
3600   ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3601 }
3602 
3603 void MacroAssembler::load_method_holder(Register holder, Register method) {
3604   ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
3605   ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
3606   ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3607 }
3608 








3609 void MacroAssembler::load_klass(Register dst, Register src) {
3610   if (UseCompressedClassPointers) {
3611     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3612     decode_klass_not_null(dst);
3613   } else {
3614     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3615   }
3616 }
3617 
3618 // ((OopHandle)result).resolve();
3619 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3620   // OopHandle::resolve is an indirection.
3621   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3622 }
3623 
3624 // ((WeakHandle)result).resolve();
3625 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3626   assert_different_registers(rresult, rtmp);
3627   Label resolved;
3628 

3648 
3649 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3650   if (UseCompressedClassPointers) {
3651     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3652     if (CompressedKlassPointers::base() == NULL) {
3653       cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3654       return;
3655     } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3656                && CompressedKlassPointers::shift() == 0) {
3657       // Only the bottom 32 bits matter
3658       cmpw(trial_klass, tmp);
3659       return;
3660     }
3661     decode_klass_not_null(tmp);
3662   } else {
3663     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3664   }
3665   cmp(trial_klass, tmp);
3666 }
3667 





3668 void MacroAssembler::store_klass(Register dst, Register src) {
3669   // FIXME: Should this be a store release?  concurrent gcs assumes
3670   // klass length is valid if klass field is not null.
3671   if (UseCompressedClassPointers) {
3672     encode_klass_not_null(src);
3673     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3674   } else {
3675     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3676   }
3677 }
3678 
3679 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3680   if (UseCompressedClassPointers) {
3681     // Store to klass gap in destination
3682     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3683   }
3684 }
3685 
3686 // Algorithm must match CompressedOops::encode.
3687 void MacroAssembler::encode_heap_oop(Register d, Register s) {

3961   narrowKlass nk = CompressedKlassPointers::encode(k);
3962   movz(dst, (nk >> 16), 16);
3963   movk(dst, nk & 0xffff);
3964 }
3965 
3966 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
3967                                     Register dst, Address src,
3968                                     Register tmp1, Register thread_tmp) {
3969   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
3970   decorators = AccessInternal::decorator_fixup(decorators);
3971   bool as_raw = (decorators & AS_RAW) != 0;
3972   if (as_raw) {
3973     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
3974   } else {
3975     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
3976   }
3977 }
3978 
3979 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
3980                                      Address dst, Register src,
3981                                      Register tmp1, Register thread_tmp) {

3982   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
3983   decorators = AccessInternal::decorator_fixup(decorators);
3984   bool as_raw = (decorators & AS_RAW) != 0;
3985   if (as_raw) {
3986     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, thread_tmp);






















3987   } else {
3988     bs->store_at(this, decorators, type, dst, src, tmp1, thread_tmp);
3989   }
3990 }
3991 


















3992 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
3993                                    Register thread_tmp, DecoratorSet decorators) {
3994   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
3995 }
3996 
3997 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
3998                                             Register thread_tmp, DecoratorSet decorators) {
3999   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
4000 }
4001 
4002 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
4003                                     Register thread_tmp, DecoratorSet decorators) {
4004   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
4005 }
4006 
4007 // Used for storing NULLs.
4008 void MacroAssembler::store_heap_oop_null(Address dst) {
4009   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg);
4010 }
4011 
4012 Address MacroAssembler::allocate_metadata_address(Metadata* obj) {
4013   assert(oop_recorder() != NULL, "this assembler needs a Recorder");
4014   int index = oop_recorder()->allocate_metadata_index(obj);
4015   RelocationHolder rspec = metadata_Relocation::spec(index);
4016   return Address((address)obj, rspec);
4017 }
4018 
4019 // Move an oop into a register.  immediate is true if we want
4020 // immediate instructions and nmethod entry barriers are not enabled.
4021 // i.e. we are not going to patch this instruction while the code is being
4022 // executed by another thread.
4023 void MacroAssembler::movoop(Register dst, jobject obj, bool immediate) {
4024   int oop_index;
4025   if (obj == NULL) {
4026     oop_index = oop_recorder()->allocate_oop_index(obj);
4027   } else {
4028 #ifdef ASSERT
4029     {

4053     oop_index = oop_recorder()->allocate_metadata_index(obj);
4054   } else {
4055     oop_index = oop_recorder()->find_index(obj);
4056   }
4057   RelocationHolder rspec = metadata_Relocation::spec(oop_index);
4058   mov(dst, Address((address)obj, rspec));
4059 }
4060 
4061 Address MacroAssembler::constant_oop_address(jobject obj) {
4062 #ifdef ASSERT
4063   {
4064     ThreadInVMfromUnknown tiv;
4065     assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
4066     assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "not an oop");
4067   }
4068 #endif
4069   int oop_index = oop_recorder()->find_index(obj);
4070   return Address((address)obj, oop_Relocation::spec(oop_index));
4071 }
4072 












































































































4073 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4074 void MacroAssembler::tlab_allocate(Register obj,
4075                                    Register var_size_in_bytes,
4076                                    int con_size_in_bytes,
4077                                    Register t1,
4078                                    Register t2,
4079                                    Label& slow_case) {
4080   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4081   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
4082 }
4083 
4084 // Defines obj, preserves var_size_in_bytes
4085 void MacroAssembler::eden_allocate(Register obj,
4086                                    Register var_size_in_bytes,
4087                                    int con_size_in_bytes,
4088                                    Register t1,
4089                                    Label& slow_case) {
4090   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4091   bs->eden_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
4092 }

4102     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4103     cmp(rscratch2, rscratch1);
4104     br(Assembler::HS, next);
4105     STOP("assert(top >= start)");
4106     should_not_reach_here();
4107 
4108     bind(next);
4109     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4110     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4111     cmp(rscratch2, rscratch1);
4112     br(Assembler::HS, ok);
4113     STOP("assert(top <= end)");
4114     should_not_reach_here();
4115 
4116     bind(ok);
4117     ldp(rscratch2, rscratch1, Address(post(sp, 16)));
4118   }
4119 #endif
4120 }
4121 













4122 // Writes to stack successive pages until offset reached to check for
4123 // stack overflow + shadow pages.  This clobbers tmp.
4124 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
4125   assert_different_registers(tmp, size, rscratch1);
4126   mov(tmp, sp);
4127   // Bang stack for total size given plus shadow page size.
4128   // Bang one page at a time because large size can bang beyond yellow and
4129   // red zones.
4130   Label loop;
4131   mov(rscratch1, os::vm_page_size());
4132   bind(loop);
4133   lea(tmp, Address(tmp, -os::vm_page_size()));
4134   subsw(size, size, rscratch1);
4135   str(size, Address(tmp));
4136   br(Assembler::GT, loop);
4137 
4138   // Bang down shadow pages too.
4139   // At this point, (tmp-0) is the last address touched, so don't
4140   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
4141   // was post-decremented.)  Skip this address by starting at i=1, and

4225   verify_cross_modify_fence_not_required();
4226 }
4227 
4228 void MacroAssembler::remove_frame(int framesize) {
4229   assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR");
4230   assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
4231   if (framesize < ((1 << 9) + 2 * wordSize)) {
4232     ldp(rfp, lr, Address(sp, framesize - 2 * wordSize));
4233     add(sp, sp, framesize);
4234   } else {
4235     if (framesize < ((1 << 12) + 2 * wordSize))
4236       add(sp, sp, framesize - 2 * wordSize);
4237     else {
4238       mov(rscratch1, framesize - 2 * wordSize);
4239       add(sp, sp, rscratch1);
4240     }
4241     ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
4242   }
4243 }
4244 



















































4245 
4246 // This method checks if provided byte array contains byte with highest bit set.
4247 address MacroAssembler::has_negatives(Register ary1, Register len, Register result) {
4248     // Simple and most common case of aligned small array which is not at the
4249     // end of memory page is placed here. All other cases are in stub.
4250     Label LOOP, END, STUB, STUB_LONG, SET_RESULT, DONE;
4251     const uint64_t UPPER_BIT_MASK=0x8080808080808080;
4252     assert_different_registers(ary1, len, result);
4253 
4254     cmpw(len, 0);
4255     br(LE, SET_RESULT);
4256     cmpw(len, 4 * wordSize);
4257     br(GE, STUB_LONG); // size > 32 then go to stub
4258 
4259     int shift = 64 - exact_log2(os::vm_page_size());
4260     lsl(rscratch1, ary1, shift);
4261     mov(rscratch2, (size_t)(4 * wordSize) << shift);
4262     adds(rscratch2, rscratch1, rscratch2);  // At end of page?
4263     br(CS, STUB); // at the end of page then go to stub
4264     subs(len, len, wordSize);

5068 //
5069 // On Linux, aarch64_get_thread_helper() clobbers only r0, r1, and flags.
5070 // On other systems, the helper is a usual C function.
5071 //
5072 void MacroAssembler::get_thread(Register dst) {
5073   RegSet saved_regs =
5074     LINUX_ONLY(RegSet::range(r0, r1)  + lr - dst)
5075     NOT_LINUX (RegSet::range(r0, r17) + lr - dst);
5076 
5077   push(saved_regs, sp);
5078 
5079   mov(lr, CAST_FROM_FN_PTR(address, JavaThread::aarch64_get_thread_helper));
5080   blr(lr);
5081   if (dst != c_rarg0) {
5082     mov(dst, c_rarg0);
5083   }
5084 
5085   pop(saved_regs, sp);
5086 }
5087 
































































































































































































































































































































































5088 void MacroAssembler::cache_wb(Address line) {
5089   assert(line.getMode() == Address::base_plus_offset, "mode should be base_plus_offset");
5090   assert(line.index() == noreg, "index should be noreg");
5091   assert(line.offset() == 0, "offset should be 0");
5092   // would like to assert this
5093   // assert(line._ext.shift == 0, "shift should be zero");
5094   if (VM_Version::features() & VM_Version::CPU_DCPOP) {
5095     // writeback using clear virtual address to point of persistence
5096     dc(Assembler::CVAP, line.base());
5097   } else {
5098     // no need to generate anything as Unsafe.writebackMemory should
5099     // never invoke this stub
5100   }
5101 }
5102 
5103 void MacroAssembler::cache_wbsync(bool is_pre) {
5104   // we only need a barrier post sync
5105   if (!is_pre) {
5106     membar(Assembler::AnyAny);
5107   }

  12  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13  * version 2 for more details (a copy is included in the LICENSE file that
  14  * accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License version
  17  * 2 along with this work; if not, write to the Free Software Foundation,
  18  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  19  *
  20  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  21  * or visit www.oracle.com if you need additional information or have any
  22  * questions.
  23  *
  24  */
  25 
  26 #include <sys/types.h>
  27 
  28 #include "precompiled.hpp"
  29 #include "jvm.h"
  30 #include "asm/assembler.hpp"
  31 #include "asm/assembler.inline.hpp"
  32 #include "ci/ciInlineKlass.hpp"
  33 #include "gc/shared/barrierSet.hpp"
  34 #include "gc/shared/barrierSetAssembler.hpp"
  35 #include "gc/shared/cardTableBarrierSet.hpp"
  36 #include "gc/shared/cardTable.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "gc/shared/tlab_globals.hpp"
  39 #include "interpreter/bytecodeHistogram.hpp"
  40 #include "interpreter/interpreter.hpp"
  41 #include "compiler/compileTask.hpp"
  42 #include "compiler/disassembler.hpp"
  43 #include "memory/resourceArea.hpp"
  44 #include "memory/universe.hpp"
  45 #include "nativeInst_aarch64.hpp"
  46 #include "oops/accessDecorators.hpp"
  47 #include "oops/compressedOops.inline.hpp"
  48 #include "oops/klass.inline.hpp"
  49 #include "runtime/icache.hpp"
  50 #include "runtime/interfaceSupport.inline.hpp"
  51 #include "runtime/jniHandles.inline.hpp"
  52 #include "runtime/sharedRuntime.hpp"
  53 #include "runtime/signature_cc.hpp"
  54 #include "runtime/stubRoutines.hpp"
  55 #include "runtime/thread.hpp"
  56 #include "utilities/powerOfTwo.hpp"
  57 #include "vmreg_aarch64.inline.hpp"
  58 #ifdef COMPILER1
  59 #include "c1/c1_LIRAssembler.hpp"
  60 #endif
  61 #ifdef COMPILER2
  62 #include "oops/oop.hpp"
  63 #include "opto/compile.hpp"
  64 #include "opto/node.hpp"
  65 #include "opto/output.hpp"
  66 #endif
  67 
  68 #ifdef PRODUCT
  69 #define BLOCK_COMMENT(str) /* nothing */
  70 #else
  71 #define BLOCK_COMMENT(str) block_comment(str)
  72 #endif
  73 #define STOP(str) stop(str);
  74 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
  75 
  76 // Patch any kind of instruction; there may be several instructions.
  77 // Return the total length (in bytes) of the instructions.

 757   ldr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
 758   str(zr, Address(java_thread, JavaThread::vm_result_offset()));
 759   verify_oop(oop_result, "broken oop in call_VM_base");
 760 }
 761 
 762 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
 763   ldr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
 764   str(zr, Address(java_thread, JavaThread::vm_result_2_offset()));
 765 }
 766 
 767 void MacroAssembler::align(int modulus) {
 768   while (offset() % modulus != 0) nop();
 769 }
 770 
 771 // these are no-ops overridden by InterpreterMacroAssembler
 772 
 773 void MacroAssembler::check_and_handle_earlyret(Register java_thread) { }
 774 
 775 void MacroAssembler::check_and_handle_popframe(Register java_thread) { }
 776 
 777 void MacroAssembler::get_default_value_oop(Register inline_klass, Register temp_reg, Register obj) {
 778 #ifdef ASSERT
 779   {
 780     Label done_check;
 781     test_klass_is_inline_type(inline_klass, temp_reg, done_check);
 782     stop("get_default_value_oop from non inline type klass");
 783     bind(done_check);
 784   }
 785 #endif
 786   Register offset = temp_reg;
 787   // Getting the offset of the pre-allocated default value
 788   ldr(offset, Address(inline_klass, in_bytes(InstanceKlass::adr_inlineklass_fixed_block_offset())));
 789   ldr(offset, Address(offset, in_bytes(InlineKlass::default_value_offset_offset())));
 790 
 791   // Getting the mirror
 792   ldr(obj, Address(inline_klass, in_bytes(Klass::java_mirror_offset())));
 793   resolve_oop_handle(obj, inline_klass);
 794 
 795   // Getting the pre-allocated default value from the mirror
 796   Address field(obj, offset);
 797   load_heap_oop(obj, field);
 798 }
 799 
 800 void MacroAssembler::get_empty_inline_type_oop(Register inline_klass, Register temp_reg, Register obj) {
 801 #ifdef ASSERT
 802   {
 803     Label done_check;
 804     test_klass_is_empty_inline_type(inline_klass, temp_reg, done_check);
 805     stop("get_empty_value from non-empty inline klass");
 806     bind(done_check);
 807   }
 808 #endif
 809   get_default_value_oop(inline_klass, temp_reg, obj);
 810 }
 811 
 812 // Look up the method for a megamorphic invokeinterface call.
 813 // The target method is determined by <intf_klass, itable_index>.
 814 // The receiver klass is in recv_klass.
 815 // On success, the result will be in method_result, and execution falls through.
 816 // On failure, execution transfers to the given label.
 817 void MacroAssembler::lookup_interface_method(Register recv_klass,
 818                                              Register intf_klass,
 819                                              RegisterOrConstant itable_index,
 820                                              Register method_result,
 821                                              Register scan_temp,
 822                                              Label& L_no_such_interface,
 823                          bool return_method) {
 824   assert_different_registers(recv_klass, intf_klass, scan_temp);
 825   assert_different_registers(method_result, intf_klass, scan_temp);
 826   assert(recv_klass != method_result || !return_method,
 827      "recv_klass can be destroyed when method isn't needed");
 828   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 829          "caller must use same register for non-constant itable index as for method");
 830 
 831   // Compute start of first itableOffsetEntry (which is at the end of the vtable)

1142   ldrb(scratch, Address(klass, InstanceKlass::init_state_offset()));
1143   subs(zr, scratch, InstanceKlass::fully_initialized);
1144   br(Assembler::EQ, *L_fast_path);
1145 
1146   // Fast path check: current thread is initializer thread
1147   ldr(scratch, Address(klass, InstanceKlass::init_thread_offset()));
1148   cmp(rthread, scratch);
1149 
1150   if (L_slow_path == &L_fallthrough) {
1151     br(Assembler::EQ, *L_fast_path);
1152     bind(*L_slow_path);
1153   } else if (L_fast_path == &L_fallthrough) {
1154     br(Assembler::NE, *L_slow_path);
1155     bind(*L_fast_path);
1156   } else {
1157     Unimplemented();
1158   }
1159 }
1160 
1161 void MacroAssembler::verify_oop(Register reg, const char* s) {
1162   if (!VerifyOops || VerifyAdapterSharing) {
1163     // Below address of the code string confuses VerifyAdapterSharing
1164     // because it may differ between otherwise equivalent adapters.
1165     return;
1166   }
1167 
1168   // Pass register number to verify_oop_subroutine
1169   const char* b = NULL;
1170   {
1171     ResourceMark rm;
1172     stringStream ss;
1173     ss.print("verify_oop: %s: %s", reg->name(), s);
1174     b = code_string(ss.as_string());
1175   }
1176   BLOCK_COMMENT("verify_oop {");
1177 
1178   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1179   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1180 
1181   mov(r0, reg);
1182   movptr(rscratch1, (uintptr_t)(address)b);
1183 
1184   // call indirectly to solve generation ordering problem
1185   lea(rscratch2, ExternalAddress(StubRoutines::verify_oop_subroutine_entry_address()));
1186   ldr(rscratch2, Address(rscratch2));
1187   blr(rscratch2);
1188 
1189   ldp(rscratch2, lr, Address(post(sp, 2 * wordSize)));
1190   ldp(r0, rscratch1, Address(post(sp, 2 * wordSize)));
1191 
1192   BLOCK_COMMENT("} verify_oop");
1193 }
1194 
1195 void MacroAssembler::verify_oop_addr(Address addr, const char* s) {
1196   if (!VerifyOops || VerifyAdapterSharing) {
1197     // Below address of the code string confuses VerifyAdapterSharing
1198     // because it may differ between otherwise equivalent adapters.
1199     return;
1200   }
1201 
1202   const char* b = NULL;
1203   {
1204     ResourceMark rm;
1205     stringStream ss;
1206     ss.print("verify_oop_addr: %s", s);
1207     b = code_string(ss.as_string());
1208   }
1209   BLOCK_COMMENT("verify_oop_addr {");
1210 
1211   stp(r0, rscratch1, Address(pre(sp, -2 * wordSize)));
1212   stp(rscratch2, lr, Address(pre(sp, -2 * wordSize)));
1213 
1214   // addr may contain sp so we will have to adjust it based on the
1215   // pushes that we just did.
1216   if (addr.uses(sp)) {
1217     lea(r0, addr);
1218     ldr(r0, Address(r0, 4 * wordSize));
1219   } else {
1220     ldr(r0, addr);

1259   stp(rscratch1, rmethod, Address(pre(sp, -2 * wordSize)));
1260 
1261   mov(rscratch1, entry_point);
1262   blr(rscratch1);
1263   if (retaddr)
1264     bind(*retaddr);
1265 
1266   ldp(rscratch1, rmethod, Address(post(sp, 2 * wordSize)));
1267 }
1268 
1269 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
1270   call_VM_leaf_base(entry_point, number_of_arguments);
1271 }
1272 
1273 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
1274   pass_arg0(this, arg_0);
1275   call_VM_leaf_base(entry_point, 1);
1276 }
1277 
1278 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1279   assert_different_registers(arg_1, c_rarg0);
1280   pass_arg0(this, arg_0);
1281   pass_arg1(this, arg_1);
1282   call_VM_leaf_base(entry_point, 2);
1283 }
1284 
1285 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
1286                                   Register arg_1, Register arg_2) {
1287   assert_different_registers(arg_1, c_rarg0);
1288   assert_different_registers(arg_2, c_rarg0, c_rarg1);
1289   pass_arg0(this, arg_0);
1290   pass_arg1(this, arg_1);
1291   pass_arg2(this, arg_2);
1292   call_VM_leaf_base(entry_point, 3);
1293 }
1294 
1295 void MacroAssembler::super_call_VM_leaf(address entry_point) {
1296   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1297 }
1298 
1299 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
1300   pass_arg0(this, arg_0);
1301   MacroAssembler::call_VM_leaf_base(entry_point, 1);
1302 }
1303 
1304 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
1305 
1306   assert(arg_0 != c_rarg1, "smashed arg");
1307   pass_arg1(this, arg_1);
1308   pass_arg0(this, arg_0);
1309   MacroAssembler::call_VM_leaf_base(entry_point, 2);
1310 }
1311 
1312 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
1313   assert(arg_0 != c_rarg2, "smashed arg");
1314   assert(arg_1 != c_rarg2, "smashed arg");
1315   pass_arg2(this, arg_2);
1316   assert(arg_0 != c_rarg1, "smashed arg");
1317   pass_arg1(this, arg_1);
1318   pass_arg0(this, arg_0);

1328   assert(arg_1 != c_rarg2, "smashed arg");
1329   pass_arg2(this, arg_2);
1330   assert(arg_0 != c_rarg1, "smashed arg");
1331   pass_arg1(this, arg_1);
1332   pass_arg0(this, arg_0);
1333   MacroAssembler::call_VM_leaf_base(entry_point, 4);
1334 }
1335 
1336 void MacroAssembler::null_check(Register reg, int offset) {
1337   if (needs_explicit_null_check(offset)) {
1338     // provoke OS NULL exception if reg = NULL by
1339     // accessing M[reg] w/o changing any registers
1340     // NOTE: this is plenty to provoke a segv
1341     ldr(zr, Address(reg));
1342   } else {
1343     // nothing to do, (later) access of M[reg + offset]
1344     // will provoke OS NULL exception if reg = NULL
1345   }
1346 }
1347 
1348 void MacroAssembler::test_markword_is_inline_type(Register markword, Label& is_inline_type) {
1349   assert_different_registers(markword, rscratch2);
1350   andr(markword, markword, markWord::inline_type_mask_in_place);
1351   mov(rscratch2, markWord::inline_type_pattern);
1352   cmp(markword, rscratch2);
1353   br(Assembler::EQ, is_inline_type);
1354 }
1355 
1356 void MacroAssembler::test_klass_is_inline_type(Register klass, Register temp_reg, Label& is_inline_type) {
1357   ldrw(temp_reg, Address(klass, Klass::access_flags_offset()));
1358   andr(temp_reg, temp_reg, JVM_ACC_INLINE);
1359   cbnz(temp_reg, is_inline_type);
1360 }
1361 
1362 void MacroAssembler::test_oop_is_not_inline_type(Register object, Register tmp, Label& not_inline_type) {
1363   cbz(object, not_inline_type);
1364   const int is_inline_type_mask = markWord::inline_type_pattern;
1365   ldr(tmp, Address(object, oopDesc::mark_offset_in_bytes()));
1366   mov(rscratch1, is_inline_type_mask);
1367   andr(tmp, tmp, rscratch1);
1368   cmp(tmp, rscratch1);
1369   br(Assembler::NE, not_inline_type);
1370 }
1371 
1372 void MacroAssembler::test_klass_is_empty_inline_type(Register klass, Register temp_reg, Label& is_empty_inline_type) {
1373 #ifdef ASSERT
1374   {
1375     Label done_check;
1376     test_klass_is_inline_type(klass, temp_reg, done_check);
1377     stop("test_klass_is_empty_inline_type with non inline type klass");
1378     bind(done_check);
1379   }
1380 #endif
1381   ldrw(temp_reg, Address(klass, InstanceKlass::misc_flags_offset()));
1382   andr(temp_reg, temp_reg, InstanceKlass::misc_flags_is_empty_inline_type());
1383   cbnz(temp_reg, is_empty_inline_type);
1384 }
1385 
1386 void MacroAssembler::test_field_is_null_free_inline_type(Register flags, Register temp_reg, Label& is_null_free_inline_type) {
1387   assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86
1388   tbnz(flags, ConstantPoolCacheEntry::is_null_free_inline_type_shift, is_null_free_inline_type);
1389 }
1390 
1391 void MacroAssembler::test_field_is_not_null_free_inline_type(Register flags, Register temp_reg, Label& not_null_free_inline_type) {
1392   assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86
1393   tbz(flags, ConstantPoolCacheEntry::is_null_free_inline_type_shift, not_null_free_inline_type);
1394 }
1395 
1396 void MacroAssembler::test_field_is_inlined(Register flags, Register temp_reg, Label& is_flattened) {
1397   assert(temp_reg == noreg, "not needed"); // keep signature uniform with x86
1398   tbnz(flags, ConstantPoolCacheEntry::is_inlined_shift, is_flattened);
1399 }
1400 
1401 void MacroAssembler::test_oop_prototype_bit(Register oop, Register temp_reg, int32_t test_bit, bool jmp_set, Label& jmp_label) {
1402   Label test_mark_word;
1403   // load mark word
1404   ldr(temp_reg, Address(oop, oopDesc::mark_offset_in_bytes()));
1405   // check displaced
1406   tst(temp_reg, markWord::unlocked_value);
1407   br(Assembler::NE, test_mark_word);
1408   // slow path use klass prototype
1409   load_prototype_header(temp_reg, oop);
1410 
1411   bind(test_mark_word);
1412   andr(temp_reg, temp_reg, test_bit);
1413   if (jmp_set) {
1414     cbnz(temp_reg, jmp_label);
1415   } else {
1416     cbz(temp_reg, jmp_label);
1417   }
1418 }
1419 
1420 void MacroAssembler::test_flattened_array_oop(Register oop, Register temp_reg, Label& is_flattened_array) {
1421   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, true, is_flattened_array);
1422 }
1423 
1424 void MacroAssembler::test_non_flattened_array_oop(Register oop, Register temp_reg,
1425                                                   Label&is_non_flattened_array) {
1426   test_oop_prototype_bit(oop, temp_reg, markWord::flat_array_bit_in_place, false, is_non_flattened_array);
1427 }
1428 
1429 void MacroAssembler::test_null_free_array_oop(Register oop, Register temp_reg, Label& is_null_free_array) {
1430   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, true, is_null_free_array);
1431 }
1432 
1433 void MacroAssembler::test_non_null_free_array_oop(Register oop, Register temp_reg, Label&is_non_null_free_array) {
1434   test_oop_prototype_bit(oop, temp_reg, markWord::null_free_array_bit_in_place, false, is_non_null_free_array);
1435 }
1436 
1437 void MacroAssembler::test_flattened_array_layout(Register lh, Label& is_flattened_array) {
1438   tst(lh, Klass::_lh_array_tag_vt_value_bit_inplace);
1439   br(Assembler::NE, is_flattened_array);
1440 }
1441 
1442 void MacroAssembler::test_non_flattened_array_layout(Register lh, Label& is_non_flattened_array) {
1443   tst(lh, Klass::_lh_array_tag_vt_value_bit_inplace);
1444   br(Assembler::EQ, is_non_flattened_array);
1445 }
1446 
1447 void MacroAssembler::test_null_free_array_layout(Register lh, Label& is_null_free_array) {
1448   tst(lh, Klass::_lh_null_free_bit_inplace);
1449   br(Assembler::NE, is_null_free_array);
1450 }
1451 
1452 void MacroAssembler::test_non_null_free_array_layout(Register lh, Label& is_non_null_free_array) {
1453   tst(lh, Klass::_lh_null_free_bit_inplace);
1454   br(Assembler::EQ, is_non_null_free_array);
1455 }
1456 
1457 // MacroAssembler protected routines needed to implement
1458 // public methods
1459 
1460 void MacroAssembler::mov(Register r, Address dest) {
1461   code_section()->relocate(pc(), dest.rspec());
1462   uint64_t imm64 = (uint64_t)dest.target();
1463   movptr(r, imm64);
1464 }
1465 
1466 // Move a constant pointer into r.  In AArch64 mode the virtual
1467 // address space is 48 bits in size, so we only need three
1468 // instructions to create a patchable instruction sequence that can
1469 // reach anywhere.
1470 void MacroAssembler::movptr(Register r, uintptr_t imm64) {
1471 #ifndef PRODUCT
1472   {
1473     char buffer[64];
1474     snprintf(buffer, sizeof(buffer), "0x%" PRIX64, (uint64_t)imm64);
1475     block_comment(buffer);
1476   }

3751   adrp(rscratch1, src2, offset);
3752   ldr(rscratch1, Address(rscratch1, offset));
3753   cmp(src1, rscratch1);
3754 }
3755 
3756 void MacroAssembler::cmpoop(Register obj1, Register obj2) {
3757   cmp(obj1, obj2);
3758 }
3759 
3760 void MacroAssembler::load_method_holder_cld(Register rresult, Register rmethod) {
3761   load_method_holder(rresult, rmethod);
3762   ldr(rresult, Address(rresult, InstanceKlass::class_loader_data_offset()));
3763 }
3764 
3765 void MacroAssembler::load_method_holder(Register holder, Register method) {
3766   ldr(holder, Address(method, Method::const_offset()));                      // ConstMethod*
3767   ldr(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
3768   ldr(holder, Address(holder, ConstantPool::pool_holder_offset_in_bytes())); // InstanceKlass*
3769 }
3770 
3771 void MacroAssembler::load_metadata(Register dst, Register src) {
3772   if (UseCompressedClassPointers) {
3773     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3774   } else {
3775     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3776   }
3777 }
3778 
3779 void MacroAssembler::load_klass(Register dst, Register src) {
3780   if (UseCompressedClassPointers) {
3781     ldrw(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3782     decode_klass_not_null(dst);
3783   } else {
3784     ldr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
3785   }
3786 }
3787 
3788 // ((OopHandle)result).resolve();
3789 void MacroAssembler::resolve_oop_handle(Register result, Register tmp) {
3790   // OopHandle::resolve is an indirection.
3791   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp, noreg);
3792 }
3793 
3794 // ((WeakHandle)result).resolve();
3795 void MacroAssembler::resolve_weak_handle(Register rresult, Register rtmp) {
3796   assert_different_registers(rresult, rtmp);
3797   Label resolved;
3798 

3818 
3819 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp) {
3820   if (UseCompressedClassPointers) {
3821     ldrw(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3822     if (CompressedKlassPointers::base() == NULL) {
3823       cmp(trial_klass, tmp, LSL, CompressedKlassPointers::shift());
3824       return;
3825     } else if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0
3826                && CompressedKlassPointers::shift() == 0) {
3827       // Only the bottom 32 bits matter
3828       cmpw(trial_klass, tmp);
3829       return;
3830     }
3831     decode_klass_not_null(tmp);
3832   } else {
3833     ldr(tmp, Address(oop, oopDesc::klass_offset_in_bytes()));
3834   }
3835   cmp(trial_klass, tmp);
3836 }
3837 
3838 void MacroAssembler::load_prototype_header(Register dst, Register src) {
3839   load_klass(dst, src);
3840   ldr(dst, Address(dst, Klass::prototype_header_offset()));
3841 }
3842 
3843 void MacroAssembler::store_klass(Register dst, Register src) {
3844   // FIXME: Should this be a store release?  concurrent gcs assumes
3845   // klass length is valid if klass field is not null.
3846   if (UseCompressedClassPointers) {
3847     encode_klass_not_null(src);
3848     strw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3849   } else {
3850     str(src, Address(dst, oopDesc::klass_offset_in_bytes()));
3851   }
3852 }
3853 
3854 void MacroAssembler::store_klass_gap(Register dst, Register src) {
3855   if (UseCompressedClassPointers) {
3856     // Store to klass gap in destination
3857     strw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
3858   }
3859 }
3860 
3861 // Algorithm must match CompressedOops::encode.
3862 void MacroAssembler::encode_heap_oop(Register d, Register s) {

4136   narrowKlass nk = CompressedKlassPointers::encode(k);
4137   movz(dst, (nk >> 16), 16);
4138   movk(dst, nk & 0xffff);
4139 }
4140 
4141 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
4142                                     Register dst, Address src,
4143                                     Register tmp1, Register thread_tmp) {
4144   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4145   decorators = AccessInternal::decorator_fixup(decorators);
4146   bool as_raw = (decorators & AS_RAW) != 0;
4147   if (as_raw) {
4148     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4149   } else {
4150     bs->load_at(this, decorators, type, dst, src, tmp1, thread_tmp);
4151   }
4152 }
4153 
4154 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
4155                                      Address dst, Register src,
4156                                      Register tmp1, Register thread_tmp, Register tmp3) {
4157 
4158   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4159   decorators = AccessInternal::decorator_fixup(decorators);
4160   bool as_raw = (decorators & AS_RAW) != 0;
4161   if (as_raw) {
4162     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, src, tmp1, thread_tmp, tmp3);
4163   } else {
4164     bs->store_at(this, decorators, type, dst, src, tmp1, thread_tmp, tmp3);
4165   }
4166 }
4167 
4168 void MacroAssembler::access_value_copy(DecoratorSet decorators, Register src, Register dst,
4169                                        Register inline_klass) {
4170   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
4171   bs->value_copy(this, decorators, src, dst, inline_klass);
4172 }
4173 
4174 void MacroAssembler::first_field_offset(Register inline_klass, Register offset) {
4175   ldr(offset, Address(inline_klass, InstanceKlass::adr_inlineklass_fixed_block_offset()));
4176   ldrw(offset, Address(offset, InlineKlass::first_field_offset_offset()));
4177 }
4178 
4179 void MacroAssembler::data_for_oop(Register oop, Register data, Register inline_klass) {
4180   // ((address) (void*) o) + vk->first_field_offset();
4181   Register offset = (data == oop) ? rscratch1 : data;
4182   first_field_offset(inline_klass, offset);
4183   if (data == oop) {
4184     add(data, data, offset);
4185   } else {
4186     lea(data, Address(oop, offset));
4187   }
4188 }
4189 
4190 void MacroAssembler::data_for_value_array_index(Register array, Register array_klass,
4191                                                 Register index, Register data) {
4192   assert_different_registers(array, array_klass, index);
4193   assert_different_registers(rscratch1, array, index);
4194 
4195   // array->base() + (index << Klass::layout_helper_log2_element_size(lh));
4196   ldrw(rscratch1, Address(array_klass, Klass::layout_helper_offset()));
4197 
4198   // Klass::layout_helper_log2_element_size(lh)
4199   // (lh >> _lh_log2_element_size_shift) & _lh_log2_element_size_mask;
4200   lsr(rscratch1, rscratch1, Klass::_lh_log2_element_size_shift);
4201   andr(rscratch1, rscratch1, Klass::_lh_log2_element_size_mask);
4202   lslv(index, index, rscratch1);
4203 
4204   add(data, array, index);
4205   add(data, data, arrayOopDesc::base_offset_in_bytes(T_INLINE_TYPE));
4206 }
4207 
4208 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
4209                                    Register thread_tmp, DecoratorSet decorators) {
4210   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp);
4211 }
4212 
4213 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
4214                                             Register thread_tmp, DecoratorSet decorators) {
4215   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL | decorators, dst, src, tmp1, thread_tmp);
4216 }
4217 
4218 void MacroAssembler::store_heap_oop(Address dst, Register src, Register tmp1,
4219                                     Register thread_tmp, Register tmp3, DecoratorSet decorators) {
4220   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, thread_tmp, tmp3);
4221 }
4222 
4223 // Used for storing NULLs.
4224 void MacroAssembler::store_heap_oop_null(Address dst) {
4225   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
4226 }
4227 
4228 Address MacroAssembler::allocate_metadata_address(Metadata* obj) {
4229   assert(oop_recorder() != NULL, "this assembler needs a Recorder");
4230   int index = oop_recorder()->allocate_metadata_index(obj);
4231   RelocationHolder rspec = metadata_Relocation::spec(index);
4232   return Address((address)obj, rspec);
4233 }
4234 
4235 // Move an oop into a register.  immediate is true if we want
4236 // immediate instructions and nmethod entry barriers are not enabled.
4237 // i.e. we are not going to patch this instruction while the code is being
4238 // executed by another thread.
4239 void MacroAssembler::movoop(Register dst, jobject obj, bool immediate) {
4240   int oop_index;
4241   if (obj == NULL) {
4242     oop_index = oop_recorder()->allocate_oop_index(obj);
4243   } else {
4244 #ifdef ASSERT
4245     {

4269     oop_index = oop_recorder()->allocate_metadata_index(obj);
4270   } else {
4271     oop_index = oop_recorder()->find_index(obj);
4272   }
4273   RelocationHolder rspec = metadata_Relocation::spec(oop_index);
4274   mov(dst, Address((address)obj, rspec));
4275 }
4276 
4277 Address MacroAssembler::constant_oop_address(jobject obj) {
4278 #ifdef ASSERT
4279   {
4280     ThreadInVMfromUnknown tiv;
4281     assert(oop_recorder() != NULL, "this assembler needs an OopRecorder");
4282     assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "not an oop");
4283   }
4284 #endif
4285   int oop_index = oop_recorder()->find_index(obj);
4286   return Address((address)obj, oop_Relocation::spec(oop_index));
4287 }
4288 
4289 // Object / value buffer allocation...
4290 void MacroAssembler::allocate_instance(Register klass, Register new_obj,
4291                                        Register t1, Register t2,
4292                                        bool clear_fields, Label& alloc_failed)
4293 {
4294   Label done, initialize_header, initialize_object, slow_case, slow_case_no_pop;
4295   Register layout_size = t1;
4296   assert(new_obj == r0, "needs to be r0, according to barrier asm eden_allocate");
4297   assert_different_registers(klass, new_obj, t1, t2);
4298 
4299   // get instance_size in InstanceKlass (scaled to a count of bytes)
4300   ldrw(layout_size, Address(klass, Klass::layout_helper_offset()));
4301   // test to see if it has a finalizer or is malformed in some way
4302   tst(layout_size, Klass::_lh_instance_slow_path_bit);
4303   br(Assembler::NE, slow_case_no_pop);
4304 
4305   // Allocate the instance:
4306   //  If TLAB is enabled:
4307   //    Try to allocate in the TLAB.
4308   //    If fails, go to the slow path.
4309   //  Else If inline contiguous allocations are enabled:
4310   //    Try to allocate in eden.
4311   //    If fails due to heap end, go to slow path.
4312   //
4313   //  If TLAB is enabled OR inline contiguous is enabled:
4314   //    Initialize the allocation.
4315   //    Exit.
4316   //
4317   //  Go to slow path.
4318   const bool allow_shared_alloc =
4319     Universe::heap()->supports_inline_contig_alloc();
4320 
4321   push(klass);
4322 
4323   if (UseTLAB) {
4324     tlab_allocate(new_obj, layout_size, 0, klass, t2, slow_case);
4325     if (ZeroTLAB || (!clear_fields)) {
4326       // the fields have been already cleared
4327       b(initialize_header);
4328     } else {
4329       // initialize both the header and fields
4330       b(initialize_object);
4331     }
4332   } else {
4333     // Allocation in the shared Eden, if allowed.
4334     //
4335     eden_allocate(new_obj, layout_size, 0, t2, slow_case);
4336   }
4337 
4338   // If UseTLAB or allow_shared_alloc are true, the object is created above and
4339   // there is an initialize need. Otherwise, skip and go to the slow path.
4340   if (UseTLAB || allow_shared_alloc) {
4341     if (clear_fields) {
4342       // The object is initialized before the header.  If the object size is
4343       // zero, go directly to the header initialization.
4344       bind(initialize_object);
4345       subs(layout_size, layout_size, sizeof(oopDesc));
4346       br(Assembler::EQ, initialize_header);
4347 
4348       // Initialize topmost object field, divide size by 8, check if odd and
4349       // test if zero.
4350 
4351   #ifdef ASSERT
4352       // make sure instance_size was multiple of 8
4353       Label L;
4354       tst(layout_size, 7);
4355       br(Assembler::EQ, L);
4356       stop("object size is not multiple of 8 - adjust this code");
4357       bind(L);
4358       // must be > 0, no extra check needed here
4359   #endif
4360 
4361       lsr(layout_size, layout_size, LogBytesPerLong);
4362 
4363       // initialize remaining object fields: instance_size was a multiple of 8
4364       {
4365         Label loop;
4366         Register base = t2;
4367 
4368         bind(loop);
4369         add(rscratch1, new_obj, layout_size, Assembler::LSL, LogBytesPerLong);
4370         str(zr, Address(rscratch1, sizeof(oopDesc) - 1*oopSize));
4371         subs(layout_size, layout_size, 1);
4372         br(Assembler::NE, loop);
4373       }
4374     } // clear_fields
4375 
4376     // initialize object header only.
4377     bind(initialize_header);
4378     pop(klass);
4379     Register mark_word = t2;
4380     ldr(mark_word, Address(klass, Klass::prototype_header_offset()));
4381     str(mark_word, Address(new_obj, oopDesc::mark_offset_in_bytes ()));
4382     store_klass_gap(new_obj, zr);  // zero klass gap for compressed oops
4383     mov(t2, klass);         // preserve klass
4384     store_klass(new_obj, t2);  // src klass reg is potentially compressed
4385 
4386     b(done);
4387   }
4388 
4389   bind(slow_case);
4390   pop(klass);
4391   bind(slow_case_no_pop);
4392   b(alloc_failed);
4393 
4394   bind(done);
4395 }
4396 
4397 // Defines obj, preserves var_size_in_bytes, okay for t2 == var_size_in_bytes.
4398 void MacroAssembler::tlab_allocate(Register obj,
4399                                    Register var_size_in_bytes,
4400                                    int con_size_in_bytes,
4401                                    Register t1,
4402                                    Register t2,
4403                                    Label& slow_case) {
4404   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4405   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, t2, slow_case);
4406 }
4407 
4408 // Defines obj, preserves var_size_in_bytes
4409 void MacroAssembler::eden_allocate(Register obj,
4410                                    Register var_size_in_bytes,
4411                                    int con_size_in_bytes,
4412                                    Register t1,
4413                                    Label& slow_case) {
4414   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
4415   bs->eden_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, t1, slow_case);
4416 }

4426     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_start_offset())));
4427     cmp(rscratch2, rscratch1);
4428     br(Assembler::HS, next);
4429     STOP("assert(top >= start)");
4430     should_not_reach_here();
4431 
4432     bind(next);
4433     ldr(rscratch2, Address(rthread, in_bytes(JavaThread::tlab_end_offset())));
4434     ldr(rscratch1, Address(rthread, in_bytes(JavaThread::tlab_top_offset())));
4435     cmp(rscratch2, rscratch1);
4436     br(Assembler::HS, ok);
4437     STOP("assert(top <= end)");
4438     should_not_reach_here();
4439 
4440     bind(ok);
4441     ldp(rscratch2, rscratch1, Address(post(sp, 16)));
4442   }
4443 #endif
4444 }
4445 
4446 void MacroAssembler::get_inline_type_field_klass(Register klass, Register index, Register inline_klass) {
4447   ldr(inline_klass, Address(klass, InstanceKlass::inline_type_field_klasses_offset()));
4448 #ifdef ASSERT
4449   {
4450     Label done;
4451     cbnz(inline_klass, done);
4452     stop("get_inline_type_field_klass contains no inline klass");
4453     bind(done);
4454   }
4455 #endif
4456   ldr(inline_klass, Address(inline_klass, index, Address::lsl(3)));
4457 }
4458 
4459 // Writes to stack successive pages until offset reached to check for
4460 // stack overflow + shadow pages.  This clobbers tmp.
4461 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
4462   assert_different_registers(tmp, size, rscratch1);
4463   mov(tmp, sp);
4464   // Bang stack for total size given plus shadow page size.
4465   // Bang one page at a time because large size can bang beyond yellow and
4466   // red zones.
4467   Label loop;
4468   mov(rscratch1, os::vm_page_size());
4469   bind(loop);
4470   lea(tmp, Address(tmp, -os::vm_page_size()));
4471   subsw(size, size, rscratch1);
4472   str(size, Address(tmp));
4473   br(Assembler::GT, loop);
4474 
4475   // Bang down shadow pages too.
4476   // At this point, (tmp-0) is the last address touched, so don't
4477   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
4478   // was post-decremented.)  Skip this address by starting at i=1, and

4562   verify_cross_modify_fence_not_required();
4563 }
4564 
4565 void MacroAssembler::remove_frame(int framesize) {
4566   assert(framesize >= 2 * wordSize, "framesize must include space for FP/LR");
4567   assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
4568   if (framesize < ((1 << 9) + 2 * wordSize)) {
4569     ldp(rfp, lr, Address(sp, framesize - 2 * wordSize));
4570     add(sp, sp, framesize);
4571   } else {
4572     if (framesize < ((1 << 12) + 2 * wordSize))
4573       add(sp, sp, framesize - 2 * wordSize);
4574     else {
4575       mov(rscratch1, framesize - 2 * wordSize);
4576       add(sp, sp, rscratch1);
4577     }
4578     ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
4579   }
4580 }
4581 
4582 void MacroAssembler::remove_frame(int initial_framesize, bool needs_stack_repair) {
4583   if (needs_stack_repair) {
4584     // Remove the extension of the caller's frame used for inline type unpacking
4585     //
4586     // Right now the stack looks like this:
4587     //
4588     // | Arguments from caller     |
4589     // |---------------------------|  <-- caller's SP
4590     // | Saved LR #1               |
4591     // | Saved FP #1               |
4592     // |---------------------------|
4593     // | Extension space for       |
4594     // |   inline arg (un)packing  |
4595     // |---------------------------|  <-- start of this method's frame
4596     // | Saved LR #2               |
4597     // | Saved FP #2               |
4598     // |---------------------------|  <-- FP
4599     // | sp_inc                    |
4600     // | method locals             |
4601     // |---------------------------|  <-- SP
4602     //
4603     // There are two copies of FP and LR on the stack. They will be identical
4604     // unless the caller has been deoptimized, in which case LR #1 will be patched
4605     // to point at the deopt blob, and LR #2 will still point into the old method.
4606     //
4607     // The sp_inc stack slot holds the total size of the frame including the
4608     // extension space minus two words for the saved FP and LR.
4609 
4610     int sp_inc_offset = initial_framesize - 3 * wordSize;  // Immediately below saved LR and FP
4611 
4612     ldr(rscratch1, Address(sp, sp_inc_offset));
4613     add(sp, sp, rscratch1);
4614     ldp(rfp, lr, Address(post(sp, 2 * wordSize)));
4615   } else {
4616     remove_frame(initial_framesize);
4617   }
4618 }
4619 
4620 void MacroAssembler::save_stack_increment(int sp_inc, int frame_size) {
4621   int real_frame_size = frame_size + sp_inc;
4622   assert(sp_inc == 0 || sp_inc > 2*wordSize, "invalid sp_inc value");
4623   assert(real_frame_size >= 2*wordSize, "frame size must include FP/LR space");
4624   assert((real_frame_size & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
4625 
4626   int sp_inc_offset = frame_size - 3 * wordSize;  // Immediately below saved LR and FP
4627 
4628   // Subtract two words for the saved FP and LR as these will be popped
4629   // separately. See remove_frame above.
4630   mov(rscratch1, real_frame_size - 2*wordSize);
4631   str(rscratch1, Address(sp, sp_inc_offset));
4632 }
4633 
4634 // This method checks if provided byte array contains byte with highest bit set.
4635 address MacroAssembler::has_negatives(Register ary1, Register len, Register result) {
4636     // Simple and most common case of aligned small array which is not at the
4637     // end of memory page is placed here. All other cases are in stub.
4638     Label LOOP, END, STUB, STUB_LONG, SET_RESULT, DONE;
4639     const uint64_t UPPER_BIT_MASK=0x8080808080808080;
4640     assert_different_registers(ary1, len, result);
4641 
4642     cmpw(len, 0);
4643     br(LE, SET_RESULT);
4644     cmpw(len, 4 * wordSize);
4645     br(GE, STUB_LONG); // size > 32 then go to stub
4646 
4647     int shift = 64 - exact_log2(os::vm_page_size());
4648     lsl(rscratch1, ary1, shift);
4649     mov(rscratch2, (size_t)(4 * wordSize) << shift);
4650     adds(rscratch2, rscratch1, rscratch2);  // At end of page?
4651     br(CS, STUB); // at the end of page then go to stub
4652     subs(len, len, wordSize);

5456 //
5457 // On Linux, aarch64_get_thread_helper() clobbers only r0, r1, and flags.
5458 // On other systems, the helper is a usual C function.
5459 //
5460 void MacroAssembler::get_thread(Register dst) {
5461   RegSet saved_regs =
5462     LINUX_ONLY(RegSet::range(r0, r1)  + lr - dst)
5463     NOT_LINUX (RegSet::range(r0, r17) + lr - dst);
5464 
5465   push(saved_regs, sp);
5466 
5467   mov(lr, CAST_FROM_FN_PTR(address, JavaThread::aarch64_get_thread_helper));
5468   blr(lr);
5469   if (dst != c_rarg0) {
5470     mov(dst, c_rarg0);
5471   }
5472 
5473   pop(saved_regs, sp);
5474 }
5475 
5476 #ifdef COMPILER2
5477 // C2 compiled method's prolog code
5478 // Moved here from aarch64.ad to support Valhalla code belows
5479 void MacroAssembler::verified_entry(Compile* C, int sp_inc) {
5480 
5481   // n.b. frame size includes space for return pc and rfp
5482   const long framesize = C->output()->frame_size_in_bytes();
5483 
5484   // insert a nop at the start of the prolog so we can patch in a
5485   // branch if we need to invalidate the method later
5486   nop();
5487 
5488   int bangsize = C->output()->bang_size_in_bytes();
5489   if (C->output()->need_stack_bang(bangsize))
5490     generate_stack_overflow_check(bangsize);
5491 
5492   build_frame(framesize);
5493 
5494   if (C->needs_stack_repair()) {
5495     save_stack_increment(sp_inc, framesize);
5496   }
5497 
5498   if (VerifyStackAtCalls) {
5499     Unimplemented();
5500   }
5501 }
5502 #endif // COMPILER2
5503 
5504 int MacroAssembler::store_inline_type_fields_to_buf(ciInlineKlass* vk, bool from_interpreter) {
5505   // An inline type might be returned. If fields are in registers we
5506   // need to allocate an inline type instance and initialize it with
5507   // the value of the fields.
5508   Label skip;
5509   // We only need a new buffered inline type if a new one is not returned
5510   tbz(r0, 0, skip);
5511   int call_offset = -1;
5512 
5513   // Be careful not to clobber r1-7 which hold returned fields
5514   // Also do not use callee-saved registers as these may be live in the interpreter
5515   Register tmp1 = r13, tmp2 = r14, klass = r15, r0_preserved = r12;
5516 
5517   // The following code is similar to allocate_instance but has some slight differences,
5518   // e.g. object size is always not zero, sometimes it's constant; storing klass ptr after
5519   // allocating is not necessary if vk != NULL, etc. allocate_instance is not aware of these.
5520   Label slow_case;
5521   // 1. Try to allocate a new buffered inline instance either from TLAB or eden space
5522   mov(r0_preserved, r0); // save r0 for slow_case since *_allocate may corrupt it when allocation failed
5523 
5524   if (vk != NULL) {
5525     // Called from C1, where the return type is statically known.
5526     movptr(klass, (intptr_t)vk->get_InlineKlass());
5527     jint obj_size = vk->layout_helper();
5528     assert(obj_size != Klass::_lh_neutral_value, "inline class in return type must have been resolved");
5529     if (UseTLAB) {
5530       tlab_allocate(r0, noreg, obj_size, tmp1, tmp2, slow_case);
5531     } else {
5532       eden_allocate(r0, noreg, obj_size, tmp1, slow_case);
5533     }
5534   } else {
5535     // Call from interpreter. R0 contains ((the InlineKlass* of the return type) | 0x01)
5536     andr(klass, r0, -2);
5537     ldrw(tmp2, Address(klass, Klass::layout_helper_offset()));
5538     if (UseTLAB) {
5539       tlab_allocate(r0, tmp2, 0, tmp1, tmp2, slow_case);
5540     } else {
5541       eden_allocate(r0, tmp2, 0, tmp1, slow_case);
5542     }
5543   }
5544   if (UseTLAB || Universe::heap()->supports_inline_contig_alloc()) {
5545     // 2. Initialize buffered inline instance header
5546     Register buffer_obj = r0;
5547     mov(rscratch1, (intptr_t)markWord::inline_type_prototype().value());
5548     str(rscratch1, Address(buffer_obj, oopDesc::mark_offset_in_bytes()));
5549     store_klass_gap(buffer_obj, zr);
5550     if (vk == NULL) {
5551       // store_klass corrupts klass, so save it for later use (interpreter case only).
5552       mov(tmp1, klass);
5553     }
5554     store_klass(buffer_obj, klass);
5555     // 3. Initialize its fields with an inline class specific handler
5556     if (vk != NULL) {
5557       far_call(RuntimeAddress(vk->pack_handler())); // no need for call info as this will not safepoint.
5558     } else {
5559       // tmp1 holds klass preserved above
5560       ldr(tmp1, Address(tmp1, InstanceKlass::adr_inlineklass_fixed_block_offset()));
5561       ldr(tmp1, Address(tmp1, InlineKlass::pack_handler_offset()));
5562       blr(tmp1);
5563     }
5564 
5565     membar(Assembler::StoreStore);
5566     b(skip);
5567   } else {
5568     // Must have already branched to slow_case in eden_allocate() above.
5569     DEBUG_ONLY(should_not_reach_here());
5570   }
5571   bind(slow_case);
5572   // We failed to allocate a new inline type, fall back to a runtime
5573   // call. Some oop field may be live in some registers but we can't
5574   // tell. That runtime call will take care of preserving them
5575   // across a GC if there's one.
5576   mov(r0, r0_preserved);
5577 
5578   if (from_interpreter) {
5579     super_call_VM_leaf(StubRoutines::store_inline_type_fields_to_buf());
5580   } else {
5581     far_call(RuntimeAddress(StubRoutines::store_inline_type_fields_to_buf()));
5582     call_offset = offset();
5583   }
5584   membar(Assembler::StoreStore);
5585 
5586   bind(skip);
5587   return call_offset;
5588 }
5589 
5590 // Move a value between registers/stack slots and update the reg_state
5591 bool MacroAssembler::move_helper(VMReg from, VMReg to, BasicType bt, RegState reg_state[]) {
5592   assert(from->is_valid() && to->is_valid(), "source and destination must be valid");
5593   if (reg_state[to->value()] == reg_written) {
5594     return true; // Already written
5595   }
5596 
5597   if (from != to && bt != T_VOID) {
5598     if (reg_state[to->value()] == reg_readonly) {
5599       return false; // Not yet writable
5600     }
5601     if (from->is_reg()) {
5602       if (to->is_reg()) {
5603         if (from->is_Register() && to->is_Register()) {
5604           mov(to->as_Register(), from->as_Register());
5605         } else if (from->is_FloatRegister() && to->is_FloatRegister()) {
5606           fmovd(to->as_FloatRegister(), from->as_FloatRegister());
5607         } else {
5608           ShouldNotReachHere();
5609         }
5610       } else {
5611         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size;
5612         Address to_addr = Address(sp, st_off);
5613         if (from->is_FloatRegister()) {
5614           if (bt == T_DOUBLE) {
5615              strd(from->as_FloatRegister(), to_addr);
5616           } else {
5617              assert(bt == T_FLOAT, "must be float");
5618              strs(from->as_FloatRegister(), to_addr);
5619           }
5620         } else {
5621           str(from->as_Register(), to_addr);
5622         }
5623       }
5624     } else {
5625       Address from_addr = Address(sp, from->reg2stack() * VMRegImpl::stack_slot_size);
5626       if (to->is_reg()) {
5627         if (to->is_FloatRegister()) {
5628           if (bt == T_DOUBLE) {
5629             ldrd(to->as_FloatRegister(), from_addr);
5630           } else {
5631             assert(bt == T_FLOAT, "must be float");
5632             ldrs(to->as_FloatRegister(), from_addr);
5633           }
5634         } else {
5635           ldr(to->as_Register(), from_addr);
5636         }
5637       } else {
5638         int st_off = to->reg2stack() * VMRegImpl::stack_slot_size;
5639         ldr(rscratch1, from_addr);
5640         str(rscratch1, Address(sp, st_off));
5641       }
5642     }
5643   }
5644 
5645   // Update register states
5646   reg_state[from->value()] = reg_writable;
5647   reg_state[to->value()] = reg_written;
5648   return true;
5649 }
5650 
5651 // Calculate the extra stack space required for packing or unpacking inline
5652 // args and adjust the stack pointer
5653 int MacroAssembler::extend_stack_for_inline_args(int args_on_stack) {
5654   int sp_inc = args_on_stack * VMRegImpl::stack_slot_size;
5655   sp_inc = align_up(sp_inc, StackAlignmentInBytes);
5656   assert(sp_inc > 0, "sanity");
5657 
5658   // Save a copy of the FP and LR here for deoptimization patching and frame walking
5659   stp(rfp, lr, Address(pre(sp, -2 * wordSize)));
5660 
5661   // Adjust the stack pointer. This will be repaired on return by MacroAssembler::remove_frame
5662   if (sp_inc < (1 << 9)) {
5663     sub(sp, sp, sp_inc);   // Fits in an immediate
5664   } else {
5665     mov(rscratch1, sp_inc);
5666     sub(sp, sp, rscratch1);
5667   }
5668 
5669   return sp_inc + 2 * wordSize;  // Account for the FP/LR space
5670 }
5671 
5672 // Read all fields from an inline type oop and store the values in registers/stack slots
5673 bool MacroAssembler::unpack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index,
5674                                           VMReg from, int& from_index, VMRegPair* to, int to_count, int& to_index,
5675                                           RegState reg_state[]) {
5676   assert(sig->at(sig_index)._bt == T_VOID, "should be at end delimiter");
5677   assert(from->is_valid(), "source must be valid");
5678   Register tmp1 = r10, tmp2 = r11;
5679   Register fromReg;
5680   if (from->is_reg()) {
5681     fromReg = from->as_Register();
5682   } else {
5683     int st_off = from->reg2stack() * VMRegImpl::stack_slot_size;
5684     ldr(tmp1, Address(sp, st_off));
5685     fromReg = tmp1;
5686   }
5687 
5688   ScalarizedInlineArgsStream stream(sig, sig_index, to, to_count, to_index, -1);
5689   bool done = true;
5690   bool mark_done = true;
5691   VMReg toReg;
5692   BasicType bt;
5693   while (stream.next(toReg, bt)) {
5694     assert(toReg->is_valid(), "destination must be valid");
5695     int off = sig->at(stream.sig_index())._offset;
5696     assert(off > 0, "offset in object should be positive");
5697     Address fromAddr = Address(fromReg, off);
5698 
5699     int idx = (int)toReg->value();
5700     if (reg_state[idx] == reg_readonly) {
5701       if (idx != from->value()) {
5702         mark_done = false;
5703       }
5704       done = false;
5705       continue;
5706     } else if (reg_state[idx] == reg_written) {
5707       continue;
5708     } else {
5709       assert(reg_state[idx] == reg_writable, "must be writable");
5710       reg_state[idx] = reg_written;
5711     }
5712 
5713     if (!toReg->is_FloatRegister()) {
5714       Register dst = toReg->is_stack() ? tmp2 : toReg->as_Register();
5715       if (is_reference_type(bt)) {
5716         load_heap_oop(dst, fromAddr);
5717       } else {
5718         bool is_signed = (bt != T_CHAR) && (bt != T_BOOLEAN);
5719         load_sized_value(dst, fromAddr, type2aelembytes(bt), is_signed);
5720       }
5721       if (toReg->is_stack()) {
5722         int st_off = toReg->reg2stack() * VMRegImpl::stack_slot_size;
5723         str(dst, Address(sp, st_off));
5724       }
5725     } else if (bt == T_DOUBLE) {
5726       ldrd(toReg->as_FloatRegister(), fromAddr);
5727     } else {
5728       assert(bt == T_FLOAT, "must be float");
5729       ldrs(toReg->as_FloatRegister(), fromAddr);
5730     }
5731   }
5732   sig_index = stream.sig_index();
5733   to_index = stream.regs_index();
5734 
5735   if (mark_done && reg_state[from->value()] != reg_written) {
5736     // This is okay because no one else will write to that slot
5737     reg_state[from->value()] = reg_writable;
5738   }
5739   from_index--;
5740   return done;
5741 }
5742 
5743 // Pack fields back into an inline type oop
5744 bool MacroAssembler::pack_inline_helper(const GrowableArray<SigEntry>* sig, int& sig_index, int vtarg_index,
5745                                         VMRegPair* from, int from_count, int& from_index, VMReg to,
5746                                         RegState reg_state[], Register val_array) {
5747   assert(sig->at(sig_index)._bt == T_INLINE_TYPE, "should be at end delimiter");
5748   assert(to->is_valid(), "destination must be valid");
5749 
5750   if (reg_state[to->value()] == reg_written) {
5751     skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
5752     return true; // Already written
5753   }
5754 
5755   // The GC barrier expanded by store_heap_oop below may call into the
5756   // runtime so use callee-saved registers for any values that need to be
5757   // preserved. The GC barrier assembler should take care of saving the
5758   // Java argument registers.
5759   Register val_obj_tmp = r21;
5760   Register from_reg_tmp = r22;
5761   Register tmp1 = r14;
5762   Register tmp2 = r13;
5763   Register tmp3 = r12;
5764   Register val_obj = to->is_stack() ? val_obj_tmp : to->as_Register();
5765 
5766   assert_different_registers(val_obj_tmp, from_reg_tmp, tmp1, tmp2, tmp3, val_array);
5767 
5768   if (reg_state[to->value()] == reg_readonly) {
5769     if (!is_reg_in_unpacked_fields(sig, sig_index, to, from, from_count, from_index)) {
5770       skip_unpacked_fields(sig, sig_index, from, from_count, from_index);
5771       return false; // Not yet writable
5772     }
5773     val_obj = val_obj_tmp;
5774   }
5775 
5776   int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + vtarg_index * type2aelembytes(T_INLINE_TYPE);
5777   load_heap_oop(val_obj, Address(val_array, index));
5778 
5779   ScalarizedInlineArgsStream stream(sig, sig_index, from, from_count, from_index);
5780   VMReg fromReg;
5781   BasicType bt;
5782   while (stream.next(fromReg, bt)) {
5783     assert(fromReg->is_valid(), "source must be valid");
5784     int off = sig->at(stream.sig_index())._offset;
5785     assert(off > 0, "offset in object should be positive");
5786     size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
5787 
5788     // Pack the scalarized field into the value object.
5789     Address dst(val_obj, off);
5790 
5791     if (!fromReg->is_FloatRegister()) {
5792       Register src;
5793       if (fromReg->is_stack()) {
5794         src = from_reg_tmp;
5795         int ld_off = fromReg->reg2stack() * VMRegImpl::stack_slot_size;
5796         load_sized_value(src, Address(sp, ld_off), size_in_bytes, /* is_signed */ false);
5797       } else {
5798         src = fromReg->as_Register();
5799       }
5800       assert_different_registers(dst.base(), src, tmp1, tmp2, tmp3, val_array);
5801       if (is_reference_type(bt)) {
5802         store_heap_oop(dst, src, tmp1, tmp2, tmp3, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
5803       } else {
5804         store_sized_value(dst, src, size_in_bytes);
5805       }
5806     } else if (bt == T_DOUBLE) {
5807       strd(fromReg->as_FloatRegister(), dst);
5808     } else {
5809       assert(bt == T_FLOAT, "must be float");
5810       strs(fromReg->as_FloatRegister(), dst);
5811     }
5812     reg_state[fromReg->value()] = reg_writable;
5813   }
5814   sig_index = stream.sig_index();
5815   from_index = stream.regs_index();
5816 
5817   assert(reg_state[to->value()] == reg_writable, "must have already been read");
5818   bool success = move_helper(val_obj->as_VMReg(), to, T_OBJECT, reg_state);
5819   assert(success, "to register must be writeable");
5820 
5821   return true;
5822 }
5823 
5824 VMReg MacroAssembler::spill_reg_for(VMReg reg) {
5825   return (reg->is_FloatRegister()) ? v0->as_VMReg() : r14->as_VMReg();
5826 }
5827 
5828 void MacroAssembler::cache_wb(Address line) {
5829   assert(line.getMode() == Address::base_plus_offset, "mode should be base_plus_offset");
5830   assert(line.index() == noreg, "index should be noreg");
5831   assert(line.offset() == 0, "offset should be 0");
5832   // would like to assert this
5833   // assert(line._ext.shift == 0, "shift should be zero");
5834   if (VM_Version::features() & VM_Version::CPU_DCPOP) {
5835     // writeback using clear virtual address to point of persistence
5836     dc(Assembler::CVAP, line.base());
5837   } else {
5838     // no need to generate anything as Unsafe.writebackMemory should
5839     // never invoke this stub
5840   }
5841 }
5842 
5843 void MacroAssembler::cache_wbsync(bool is_pre) {
5844   // we only need a barrier post sync
5845   if (!is_pre) {
5846     membar(Assembler::AnyAny);
5847   }
< prev index next >