1 /*
   2  * Copyright (c) 2003, 2022, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"
  30 #include "code/codeCache.hpp"
  31 #include "code/compiledIC.hpp"
  32 #include "code/debugInfoRec.hpp"
  33 #include "code/icBuffer.hpp"
  34 #include "code/vtableStubs.hpp"
  35 #include "compiler/oopMap.hpp"
  36 #include "gc/shared/barrierSetAssembler.hpp"
  37 #include "interpreter/interpreter.hpp"
  38 #include "interpreter/interp_masm.hpp"
  39 #include "logging/log.hpp"
  40 #include "memory/resourceArea.hpp"
  41 #include "nativeInst_aarch64.hpp"
  42 #include "oops/compiledICHolder.hpp"
  43 #include "oops/klass.inline.hpp"
  44 #include "oops/method.inline.hpp"
  45 #include "prims/methodHandles.hpp"
  46 #include "runtime/continuation.hpp"
  47 #include "runtime/continuationEntry.inline.hpp"
  48 #include "runtime/globals.hpp"
  49 #include "runtime/jniHandles.hpp"
  50 #include "runtime/safepointMechanism.hpp"
  51 #include "runtime/sharedRuntime.hpp"
  52 #include "runtime/signature.hpp"
  53 #include "runtime/stubRoutines.hpp"
  54 #include "runtime/vframeArray.hpp"
  55 #include "utilities/align.hpp"
  56 #include "utilities/formatBuffer.hpp"
  57 #include "vmreg_aarch64.inline.hpp"
  58 #ifdef COMPILER1
  59 #include "c1/c1_Runtime1.hpp"
  60 #endif
  61 #ifdef COMPILER2
  62 #include "adfiles/ad_aarch64.hpp"
  63 #include "opto/runtime.hpp"
  64 #endif
  65 #if INCLUDE_JVMCI
  66 #include "jvmci/jvmciJavaClasses.hpp"
  67 #endif
  68 
  69 #define __ masm->
  70 
  71 const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;
  72 
  73 class SimpleRuntimeFrame {
  74 
  75   public:
  76 
  77   // Most of the runtime stubs have this simple frame layout.
  78   // This class exists to make the layout shared in one place.
  79   // Offsets are for compiler stack slots, which are jints.
  80   enum layout {
  81     // The frame sender code expects that rbp will be in the "natural" place and
  82     // will override any oopMap setting for it. We must therefore force the layout
  83     // so that it agrees with the frame sender code.
  84     // we don't expect any arg reg save area so aarch64 asserts that
  85     // frame::arg_reg_save_area_bytes == 0
  86     rfp_off = 0,
  87     rfp_off2,
  88     return_off, return_off2,
  89     framesize
  90   };
  91 };
  92 
  93 // FIXME -- this is used by C1
  94 class RegisterSaver {
  95   const bool _save_vectors;
  96  public:
  97   RegisterSaver(bool save_vectors) : _save_vectors(save_vectors) {}
  98 
  99   OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words);
 100   void restore_live_registers(MacroAssembler* masm);
 101 
 102   // Offsets into the register save area
 103   // Used by deoptimization when it is managing result register
 104   // values on its own
 105 
 106   int reg_offset_in_bytes(Register r);
 107   int r0_offset_in_bytes()    { return reg_offset_in_bytes(r0); }
 108   int rscratch1_offset_in_bytes()    { return reg_offset_in_bytes(rscratch1); }
 109   int v0_offset_in_bytes();
 110 
 111   // Total stack size in bytes for saving sve predicate registers.
 112   int total_sve_predicate_in_bytes();
 113 
 114   // Capture info about frame layout
 115   // Note this is only correct when not saving full vectors.
 116   enum layout {
 117                 fpu_state_off = 0,
 118                 fpu_state_end = fpu_state_off + FPUStateSizeInWords - 1,
 119                 // The frame sender code expects that rfp will be in
 120                 // the "natural" place and will override any oopMap
 121                 // setting for it. We must therefore force the layout
 122                 // so that it agrees with the frame sender code.
 123                 r0_off = fpu_state_off + FPUStateSizeInWords,
 124                 rfp_off = r0_off + (Register::number_of_registers - 2) * Register::max_slots_per_register,
 125                 return_off = rfp_off + Register::max_slots_per_register,      // slot for return address
 126                 reg_save_size = return_off + Register::max_slots_per_register};
 127 
 128 };
 129 
 130 int RegisterSaver::reg_offset_in_bytes(Register r) {
 131   // The integer registers are located above the floating point
 132   // registers in the stack frame pushed by save_live_registers() so the
 133   // offset depends on whether we are saving full vectors, and whether
 134   // those vectors are NEON or SVE.
 135 
 136   int slots_per_vect = FloatRegister::save_slots_per_register;
 137 
 138 #if COMPILER2_OR_JVMCI
 139   if (_save_vectors) {
 140     slots_per_vect = FloatRegister::slots_per_neon_register;
 141 
 142 #ifdef COMPILER2
 143     if (Matcher::supports_scalable_vector()) {
 144       slots_per_vect = Matcher::scalable_vector_reg_size(T_FLOAT);
 145     }
 146 #endif
 147   }
 148 #endif
 149 
 150   int r0_offset = v0_offset_in_bytes() + (slots_per_vect * FloatRegister::number_of_registers) * BytesPerInt;
 151   return r0_offset + r->encoding() * wordSize;
 152 }
 153 
 154 int RegisterSaver::v0_offset_in_bytes() {
 155   // The floating point registers are located above the predicate registers if
 156   // they are present in the stack frame pushed by save_live_registers(). So the
 157   // offset depends on the saved total predicate vectors in the stack frame.
 158   return (total_sve_predicate_in_bytes() / VMRegImpl::stack_slot_size) * BytesPerInt;
 159 }
 160 
 161 int RegisterSaver::total_sve_predicate_in_bytes() {
 162 #ifdef COMPILER2
 163   if (_save_vectors && Matcher::supports_scalable_vector()) {
 164     // The number of total predicate bytes is unlikely to be a multiple
 165     // of 16 bytes so we manually align it up.
 166     return align_up(Matcher::scalable_predicate_reg_slots() *
 167                     VMRegImpl::stack_slot_size *
 168                     PRegister::number_of_saved_registers, 16);
 169   }
 170 #endif
 171   return 0;
 172 }
 173 
 174 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words) {
 175   bool use_sve = false;
 176   int sve_vector_size_in_bytes = 0;
 177   int sve_vector_size_in_slots = 0;
 178   int sve_predicate_size_in_slots = 0;
 179   int total_predicate_in_bytes = total_sve_predicate_in_bytes();
 180   int total_predicate_in_slots = total_predicate_in_bytes / VMRegImpl::stack_slot_size;
 181 
 182 #ifdef COMPILER2
 183   use_sve = Matcher::supports_scalable_vector();
 184   if (use_sve) {
 185     sve_vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 186     sve_vector_size_in_slots = Matcher::scalable_vector_reg_size(T_FLOAT);
 187     sve_predicate_size_in_slots = Matcher::scalable_predicate_reg_slots();
 188   }
 189 #endif
 190 
 191 #if COMPILER2_OR_JVMCI
 192   if (_save_vectors) {
 193     int extra_save_slots_per_register = 0;
 194     // Save upper half of vector registers
 195     if (use_sve) {
 196       extra_save_slots_per_register = sve_vector_size_in_slots - FloatRegister::save_slots_per_register;
 197     } else {
 198       extra_save_slots_per_register = FloatRegister::extra_save_slots_per_neon_register;
 199     }
 200     int extra_vector_bytes = extra_save_slots_per_register *
 201                              VMRegImpl::stack_slot_size *
 202                              FloatRegister::number_of_registers;
 203     additional_frame_words += ((extra_vector_bytes + total_predicate_in_bytes) / wordSize);
 204   }
 205 #else
 206   assert(!_save_vectors, "vectors are generated only by C2 and JVMCI");
 207 #endif
 208 
 209   int frame_size_in_bytes = align_up(additional_frame_words * wordSize +
 210                                      reg_save_size * BytesPerInt, 16);
 211   // OopMap frame size is in compiler stack slots (jint's) not bytes or words
 212   int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
 213   // The caller will allocate additional_frame_words
 214   int additional_frame_slots = additional_frame_words * wordSize / BytesPerInt;
 215   // CodeBlob frame size is in words.
 216   int frame_size_in_words = frame_size_in_bytes / wordSize;
 217   *total_frame_words = frame_size_in_words;
 218 
 219   // Save Integer and Float registers.
 220   __ enter();
 221   __ push_CPU_state(_save_vectors, use_sve, sve_vector_size_in_bytes, total_predicate_in_bytes);
 222 
 223   // Set an oopmap for the call site.  This oopmap will map all
 224   // oop-registers and debug-info registers as callee-saved.  This
 225   // will allow deoptimization at this safepoint to find all possible
 226   // debug-info recordings, as well as let GC find all oops.
 227 
 228   OopMapSet *oop_maps = new OopMapSet();
 229   OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
 230 
 231   for (int i = 0; i < Register::number_of_registers; i++) {
 232     Register r = as_Register(i);
 233     if (i <= rfp->encoding() && r != rscratch1 && r != rscratch2) {
 234       // SP offsets are in 4-byte words.
 235       // Register slots are 8 bytes wide, 32 floating-point registers.
 236       int sp_offset = Register::max_slots_per_register * i +
 237                       FloatRegister::save_slots_per_register * FloatRegister::number_of_registers;
 238       oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset + additional_frame_slots), r->as_VMReg());
 239     }
 240   }
 241 
 242   for (int i = 0; i < FloatRegister::number_of_registers; i++) {
 243     FloatRegister r = as_FloatRegister(i);
 244     int sp_offset = 0;
 245     if (_save_vectors) {
 246       sp_offset = use_sve ? (total_predicate_in_slots + sve_vector_size_in_slots * i) :
 247                             (FloatRegister::slots_per_neon_register * i);
 248     } else {
 249       sp_offset = FloatRegister::save_slots_per_register * i;
 250     }
 251     oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), r->as_VMReg());
 252   }
 253 
 254   if (_save_vectors && use_sve) {
 255     for (int i = 0; i < PRegister::number_of_saved_registers; i++) {
 256       PRegister r = as_PRegister(i);
 257       int sp_offset = sve_predicate_size_in_slots * i;
 258       oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), r->as_VMReg());
 259     }
 260   }
 261 
 262   return oop_map;
 263 }
 264 
 265 void RegisterSaver::restore_live_registers(MacroAssembler* masm) {
 266 #ifdef COMPILER2
 267   __ pop_CPU_state(_save_vectors, Matcher::supports_scalable_vector(),
 268                    Matcher::scalable_vector_reg_size(T_BYTE), total_sve_predicate_in_bytes());
 269 #else
 270 #if !INCLUDE_JVMCI
 271   assert(!_save_vectors, "vectors are generated only by C2 and JVMCI");
 272 #endif
 273   __ pop_CPU_state(_save_vectors);
 274 #endif
 275   __ ldp(rfp, lr, Address(__ post(sp, 2 * wordSize)));
 276   __ authenticate_return_address();
 277 }
 278 
 279 // Is vector's size (in bytes) bigger than a size saved by default?
 280 // 8 bytes vector registers are saved by default on AArch64.
 281 // The SVE supported min vector size is 8 bytes and we need to save
 282 // predicate registers when the vector size is 8 bytes as well.
 283 bool SharedRuntime::is_wide_vector(int size) {
 284   return size > 8 || (UseSVE > 0 && size >= 8);
 285 }
 286 
 287 // ---------------------------------------------------------------------------
 288 // Read the array of BasicTypes from a signature, and compute where the
 289 // arguments should go.  Values in the VMRegPair regs array refer to 4-byte
 290 // quantities.  Values less than VMRegImpl::stack0 are registers, those above
 291 // refer to 4-byte stack slots.  All stack slots are based off of the stack pointer
 292 // as framesizes are fixed.
 293 // VMRegImpl::stack0 refers to the first slot 0(sp).
 294 // and VMRegImpl::stack0+1 refers to the memory word 4-byes higher.
 295 // Register up to Register::number_of_registers are the 64-bit
 296 // integer registers.
 297 
 298 // Note: the INPUTS in sig_bt are in units of Java argument words,
 299 // which are 64-bit.  The OUTPUTS are in 32-bit units.
 300 
 301 // The Java calling convention is a "shifted" version of the C ABI.
 302 // By skipping the first C ABI register we can call non-static jni
 303 // methods with small numbers of arguments without having to shuffle
 304 // the arguments at all. Since we control the java ABI we ought to at
 305 // least get some advantage out of it.
 306 
 307 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
 308                                            VMRegPair *regs,
 309                                            int total_args_passed) {
 310 
 311   // Create the mapping between argument positions and
 312   // registers.
 313   static const Register INT_ArgReg[Argument::n_int_register_parameters_j] = {
 314     j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7
 315   };
 316   static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_j] = {
 317     j_farg0, j_farg1, j_farg2, j_farg3,
 318     j_farg4, j_farg5, j_farg6, j_farg7
 319   };
 320 
 321 
 322   uint int_args = 0;
 323   uint fp_args = 0;
 324   uint stk_args = 0; // inc by 2 each time
 325 
 326   for (int i = 0; i < total_args_passed; i++) {
 327     switch (sig_bt[i]) {
 328     case T_BOOLEAN:
 329     case T_CHAR:
 330     case T_BYTE:
 331     case T_SHORT:
 332     case T_INT:
 333       if (int_args < Argument::n_int_register_parameters_j) {
 334         regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
 335       } else {
 336         regs[i].set1(VMRegImpl::stack2reg(stk_args));
 337         stk_args += 2;
 338       }
 339       break;
 340     case T_VOID:
 341       // halves of T_LONG or T_DOUBLE
 342       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 343       regs[i].set_bad();
 344       break;
 345     case T_LONG:
 346       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 347       // fall through
 348     case T_OBJECT:
 349     case T_ARRAY:
 350     case T_ADDRESS:
 351       if (int_args < Argument::n_int_register_parameters_j) {
 352         regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
 353       } else {
 354         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 355         stk_args += 2;
 356       }
 357       break;
 358     case T_FLOAT:
 359       if (fp_args < Argument::n_float_register_parameters_j) {
 360         regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
 361       } else {
 362         regs[i].set1(VMRegImpl::stack2reg(stk_args));
 363         stk_args += 2;
 364       }
 365       break;
 366     case T_DOUBLE:
 367       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 368       if (fp_args < Argument::n_float_register_parameters_j) {
 369         regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 370       } else {
 371         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 372         stk_args += 2;
 373       }
 374       break;
 375     default:
 376       ShouldNotReachHere();
 377       break;
 378     }
 379   }
 380 
 381   return align_up(stk_args, 2);
 382 }
 383 
 384 // Patch the callers callsite with entry to compiled code if it exists.
 385 static void patch_callers_callsite(MacroAssembler *masm) {
 386   Label L;
 387   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 388   __ cbz(rscratch1, L);
 389 
 390   __ enter();
 391   __ push_CPU_state();
 392 
 393   // VM needs caller's callsite
 394   // VM needs target method
 395   // This needs to be a long call since we will relocate this adapter to
 396   // the codeBuffer and it may not reach
 397 
 398 #ifndef PRODUCT
 399   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
 400 #endif
 401 
 402   __ mov(c_rarg0, rmethod);
 403   __ mov(c_rarg1, lr);
 404   __ authenticate_return_address(c_rarg1, rscratch1);
 405   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 406   __ blr(rscratch1);
 407 
 408   // Explicit isb required because fixup_callers_callsite may change the code
 409   // stream.
 410   __ safepoint_isb();
 411 
 412   __ pop_CPU_state();
 413   // restore sp
 414   __ leave();
 415   __ bind(L);
 416 }
 417 
 418 static void gen_c2i_adapter(MacroAssembler *masm,
 419                             int total_args_passed,
 420                             int comp_args_on_stack,
 421                             const BasicType *sig_bt,
 422                             const VMRegPair *regs,
 423                             Label& skip_fixup) {
 424   // Before we get into the guts of the C2I adapter, see if we should be here
 425   // at all.  We've come from compiled code and are attempting to jump to the
 426   // interpreter, which means the caller made a static call to get here
 427   // (vcalls always get a compiled target if there is one).  Check for a
 428   // compiled target.  If there is one, we need to patch the caller's call.
 429   patch_callers_callsite(masm);
 430 
 431   __ bind(skip_fixup);
 432 
 433   int words_pushed = 0;
 434 
 435   // Since all args are passed on the stack, total_args_passed *
 436   // Interpreter::stackElementSize is the space we need.
 437 
 438   int extraspace = total_args_passed * Interpreter::stackElementSize;
 439 
 440   __ mov(r19_sender_sp, sp);
 441 
 442   // stack is aligned, keep it that way
 443   extraspace = align_up(extraspace, 2*wordSize);
 444 
 445   if (extraspace)
 446     __ sub(sp, sp, extraspace);
 447 
 448   // Now write the args into the outgoing interpreter space
 449   for (int i = 0; i < total_args_passed; i++) {
 450     if (sig_bt[i] == T_VOID) {
 451       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 452       continue;
 453     }
 454 
 455     // offset to start parameters
 456     int st_off   = (total_args_passed - i - 1) * Interpreter::stackElementSize;
 457     int next_off = st_off - Interpreter::stackElementSize;
 458 
 459     // Say 4 args:
 460     // i   st_off
 461     // 0   32 T_LONG
 462     // 1   24 T_VOID
 463     // 2   16 T_OBJECT
 464     // 3    8 T_BOOL
 465     // -    0 return address
 466     //
 467     // However to make thing extra confusing. Because we can fit a Java long/double in
 468     // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
 469     // leaves one slot empty and only stores to a single slot. In this case the
 470     // slot that is occupied is the T_VOID slot. See I said it was confusing.
 471 
 472     VMReg r_1 = regs[i].first();
 473     VMReg r_2 = regs[i].second();
 474     if (!r_1->is_valid()) {
 475       assert(!r_2->is_valid(), "");
 476       continue;
 477     }
 478     if (r_1->is_stack()) {
 479       // memory to memory use rscratch1
 480       int ld_off = (r_1->reg2stack() * VMRegImpl::stack_slot_size
 481                     + extraspace
 482                     + words_pushed * wordSize);
 483       if (!r_2->is_valid()) {
 484         // sign extend??
 485         __ ldrw(rscratch1, Address(sp, ld_off));
 486         __ str(rscratch1, Address(sp, st_off));
 487 
 488       } else {
 489 
 490         __ ldr(rscratch1, Address(sp, ld_off));
 491 
 492         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
 493         // T_DOUBLE and T_LONG use two slots in the interpreter
 494         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 495           // ld_off == LSW, ld_off+wordSize == MSW
 496           // st_off == MSW, next_off == LSW
 497           __ str(rscratch1, Address(sp, next_off));
 498 #ifdef ASSERT
 499           // Overwrite the unused slot with known junk
 500           __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaaaull);
 501           __ str(rscratch1, Address(sp, st_off));
 502 #endif /* ASSERT */
 503         } else {
 504           __ str(rscratch1, Address(sp, st_off));
 505         }
 506       }
 507     } else if (r_1->is_Register()) {
 508       Register r = r_1->as_Register();
 509       if (!r_2->is_valid()) {
 510         // must be only an int (or less ) so move only 32bits to slot
 511         // why not sign extend??
 512         __ str(r, Address(sp, st_off));
 513       } else {
 514         // Two VMREgs|OptoRegs can be T_OBJECT, T_ADDRESS, T_DOUBLE, T_LONG
 515         // T_DOUBLE and T_LONG use two slots in the interpreter
 516         if ( sig_bt[i] == T_LONG || sig_bt[i] == T_DOUBLE) {
 517           // jlong/double in gpr
 518 #ifdef ASSERT
 519           // Overwrite the unused slot with known junk
 520           __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaabull);
 521           __ str(rscratch1, Address(sp, st_off));
 522 #endif /* ASSERT */
 523           __ str(r, Address(sp, next_off));
 524         } else {
 525           __ str(r, Address(sp, st_off));
 526         }
 527       }
 528     } else {
 529       assert(r_1->is_FloatRegister(), "");
 530       if (!r_2->is_valid()) {
 531         // only a float use just part of the slot
 532         __ strs(r_1->as_FloatRegister(), Address(sp, st_off));
 533       } else {
 534 #ifdef ASSERT
 535         // Overwrite the unused slot with known junk
 536         __ mov(rscratch1, (uint64_t)0xdeadffffdeadaaacull);
 537         __ str(rscratch1, Address(sp, st_off));
 538 #endif /* ASSERT */
 539         __ strd(r_1->as_FloatRegister(), Address(sp, next_off));
 540       }
 541     }
 542   }
 543 
 544   __ mov(esp, sp); // Interp expects args on caller's expression stack
 545 
 546   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
 547   __ br(rscratch1);
 548 }
 549 
 550 
 551 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
 552                                     int total_args_passed,
 553                                     int comp_args_on_stack,
 554                                     const BasicType *sig_bt,
 555                                     const VMRegPair *regs) {
 556 
 557   // Note: r19_sender_sp contains the senderSP on entry. We must
 558   // preserve it since we may do a i2c -> c2i transition if we lose a
 559   // race where compiled code goes non-entrant while we get args
 560   // ready.
 561 
 562   // Adapters are frameless.
 563 
 564   // An i2c adapter is frameless because the *caller* frame, which is
 565   // interpreted, routinely repairs its own esp (from
 566   // interpreter_frame_last_sp), even if a callee has modified the
 567   // stack pointer.  It also recalculates and aligns sp.
 568 
 569   // A c2i adapter is frameless because the *callee* frame, which is
 570   // interpreted, routinely repairs its caller's sp (from sender_sp,
 571   // which is set up via the senderSP register).
 572 
 573   // In other words, if *either* the caller or callee is interpreted, we can
 574   // get the stack pointer repaired after a call.
 575 
 576   // This is why c2i and i2c adapters cannot be indefinitely composed.
 577   // In particular, if a c2i adapter were to somehow call an i2c adapter,
 578   // both caller and callee would be compiled methods, and neither would
 579   // clean up the stack pointer changes performed by the two adapters.
 580   // If this happens, control eventually transfers back to the compiled
 581   // caller, but with an uncorrected stack, causing delayed havoc.
 582 
 583   if (VerifyAdapterCalls &&
 584       (Interpreter::code() != NULL || StubRoutines::code1() != NULL)) {
 585 #if 0
 586     // So, let's test for cascading c2i/i2c adapters right now.
 587     //  assert(Interpreter::contains($return_addr) ||
 588     //         StubRoutines::contains($return_addr),
 589     //         "i2c adapter must return to an interpreter frame");
 590     __ block_comment("verify_i2c { ");
 591     Label L_ok;
 592     if (Interpreter::code() != NULL)
 593       range_check(masm, rax, r11,
 594                   Interpreter::code()->code_start(), Interpreter::code()->code_end(),
 595                   L_ok);
 596     if (StubRoutines::code1() != NULL)
 597       range_check(masm, rax, r11,
 598                   StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
 599                   L_ok);
 600     if (StubRoutines::code2() != NULL)
 601       range_check(masm, rax, r11,
 602                   StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
 603                   L_ok);
 604     const char* msg = "i2c adapter must return to an interpreter frame";
 605     __ block_comment(msg);
 606     __ stop(msg);
 607     __ bind(L_ok);
 608     __ block_comment("} verify_i2ce ");
 609 #endif
 610   }
 611 
 612   // Cut-out for having no stack args.
 613   int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
 614   if (comp_args_on_stack) {
 615     __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
 616     __ andr(sp, rscratch1, -16);
 617   }
 618 
 619   // Will jump to the compiled code just as if compiled code was doing it.
 620   // Pre-load the register-jump target early, to schedule it better.
 621   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_offset())));
 622 
 623 #if INCLUDE_JVMCI
 624   if (EnableJVMCI) {
 625     // check if this call should be routed towards a specific entry point
 626     __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 627     Label no_alternative_target;
 628     __ cbz(rscratch2, no_alternative_target);
 629     __ mov(rscratch1, rscratch2);
 630     __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 631     __ bind(no_alternative_target);
 632   }
 633 #endif // INCLUDE_JVMCI
 634 
 635   // Now generate the shuffle code.
 636   for (int i = 0; i < total_args_passed; i++) {
 637     if (sig_bt[i] == T_VOID) {
 638       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 639       continue;
 640     }
 641 
 642     // Pick up 0, 1 or 2 words from SP+offset.
 643 
 644     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
 645             "scrambled load targets?");
 646     // Load in argument order going down.
 647     int ld_off = (total_args_passed - i - 1)*Interpreter::stackElementSize;
 648     // Point to interpreter value (vs. tag)
 649     int next_off = ld_off - Interpreter::stackElementSize;
 650     //
 651     //
 652     //
 653     VMReg r_1 = regs[i].first();
 654     VMReg r_2 = regs[i].second();
 655     if (!r_1->is_valid()) {
 656       assert(!r_2->is_valid(), "");
 657       continue;
 658     }
 659     if (r_1->is_stack()) {
 660       // Convert stack slot to an SP offset (+ wordSize to account for return address )
 661       int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size;
 662       if (!r_2->is_valid()) {
 663         // sign extend???
 664         __ ldrsw(rscratch2, Address(esp, ld_off));
 665         __ str(rscratch2, Address(sp, st_off));
 666       } else {
 667         //
 668         // We are using two optoregs. This can be either T_OBJECT,
 669         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 670         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 671         // So we must adjust where to pick up the data to match the
 672         // interpreter.
 673         //
 674         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 675         // are accessed as negative so LSW is at LOW address
 676 
 677         // ld_off is MSW so get LSW
 678         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 679                            next_off : ld_off;
 680         __ ldr(rscratch2, Address(esp, offset));
 681         // st_off is LSW (i.e. reg.first())
 682         __ str(rscratch2, Address(sp, st_off));
 683       }
 684     } else if (r_1->is_Register()) {  // Register argument
 685       Register r = r_1->as_Register();
 686       if (r_2->is_valid()) {
 687         //
 688         // We are using two VMRegs. This can be either T_OBJECT,
 689         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 690         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 691         // So we must adjust where to pick up the data to match the
 692         // interpreter.
 693 
 694         const int offset = (sig_bt[i]==T_LONG||sig_bt[i]==T_DOUBLE)?
 695                            next_off : ld_off;
 696 
 697         // this can be a misaligned move
 698         __ ldr(r, Address(esp, offset));
 699       } else {
 700         // sign extend and use a full word?
 701         __ ldrw(r, Address(esp, ld_off));
 702       }
 703     } else {
 704       if (!r_2->is_valid()) {
 705         __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
 706       } else {
 707         __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
 708       }
 709     }
 710   }
 711 
 712   __ mov(rscratch2, rscratch1);
 713   __ push_cont_fastpath(rthread); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about; kills rscratch1
 714   __ mov(rscratch1, rscratch2);
 715 
 716   // 6243940 We might end up in handle_wrong_method if
 717   // the callee is deoptimized as we race thru here. If that
 718   // happens we don't want to take a safepoint because the
 719   // caller frame will look interpreted and arguments are now
 720   // "compiled" so it is much better to make this transition
 721   // invisible to the stack walking code. Unfortunately if
 722   // we try and find the callee by normal means a safepoint
 723   // is possible. So we stash the desired callee in the thread
 724   // and the vm will find there should this case occur.
 725 
 726   __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));
 727 
 728   __ br(rscratch1);
 729 }
 730 
 731 // ---------------------------------------------------------------
 732 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
 733                                                             int total_args_passed,
 734                                                             int comp_args_on_stack,
 735                                                             const BasicType *sig_bt,
 736                                                             const VMRegPair *regs,
 737                                                             AdapterFingerPrint* fingerprint) {
 738   address i2c_entry = __ pc();
 739 
 740   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 741 
 742   address c2i_unverified_entry = __ pc();
 743   Label skip_fixup;
 744 
 745   Label ok;
 746 
 747   Register holder = rscratch2;
 748   Register receiver = j_rarg0;
 749   Register tmp = r10;  // A call-clobbered register not used for arg passing
 750 
 751   // -------------------------------------------------------------------------
 752   // Generate a C2I adapter.  On entry we know rmethod holds the Method* during calls
 753   // to the interpreter.  The args start out packed in the compiled layout.  They
 754   // need to be unpacked into the interpreter layout.  This will almost always
 755   // require some stack space.  We grow the current (compiled) stack, then repack
 756   // the args.  We  finally end in a jump to the generic interpreter entry point.
 757   // On exit from the interpreter, the interpreter will restore our SP (lest the
 758   // compiled code, which relies solely on SP and not FP, get sick).
 759 
 760   {
 761     __ block_comment("c2i_unverified_entry {");
 762     __ load_klass(rscratch1, receiver);
 763     __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
 764     __ cmp(rscratch1, tmp);
 765     __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
 766     __ br(Assembler::EQ, ok);
 767     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 768 
 769     __ bind(ok);
 770     // Method might have been compiled since the call site was patched to
 771     // interpreted; if that is the case treat it as a miss so we can get
 772     // the call site corrected.
 773     __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 774     __ cbz(rscratch1, skip_fixup);
 775     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 776     __ block_comment("} c2i_unverified_entry");
 777   }
 778 
 779   address c2i_entry = __ pc();
 780 
 781   // Class initialization barrier for static methods
 782   address c2i_no_clinit_check_entry = NULL;
 783   if (VM_Version::supports_fast_class_init_checks()) {
 784     Label L_skip_barrier;
 785 
 786     { // Bypass the barrier for non-static methods
 787       __ ldrw(rscratch1, Address(rmethod, Method::access_flags_offset()));
 788       __ andsw(zr, rscratch1, JVM_ACC_STATIC);
 789       __ br(Assembler::EQ, L_skip_barrier); // non-static
 790     }
 791 
 792     __ load_method_holder(rscratch2, rmethod);
 793     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
 794     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
 795 
 796     __ bind(L_skip_barrier);
 797     c2i_no_clinit_check_entry = __ pc();
 798   }
 799 
 800   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 801   bs->c2i_entry_barrier(masm);
 802 
 803   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
 804 
 805   __ flush();
 806   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry, c2i_no_clinit_check_entry);
 807 }
 808 
 809 static int c_calling_convention_priv(const BasicType *sig_bt,
 810                                          VMRegPair *regs,
 811                                          VMRegPair *regs2,
 812                                          int total_args_passed) {
 813   assert(regs2 == NULL, "not needed on AArch64");
 814 
 815 // We return the amount of VMRegImpl stack slots we need to reserve for all
 816 // the arguments NOT counting out_preserve_stack_slots.
 817 
 818     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
 819       c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5,  c_rarg6,  c_rarg7
 820     };
 821     static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
 822       c_farg0, c_farg1, c_farg2, c_farg3,
 823       c_farg4, c_farg5, c_farg6, c_farg7
 824     };
 825 
 826     uint int_args = 0;
 827     uint fp_args = 0;
 828     uint stk_args = 0; // inc by 2 each time
 829 
 830     for (int i = 0; i < total_args_passed; i++) {
 831       switch (sig_bt[i]) {
 832       case T_BOOLEAN:
 833       case T_CHAR:
 834       case T_BYTE:
 835       case T_SHORT:
 836       case T_INT:
 837         if (int_args < Argument::n_int_register_parameters_c) {
 838           regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
 839         } else {
 840 #ifdef __APPLE__
 841           // Less-than word types are stored one after another.
 842           // The code is unable to handle this so bailout.
 843           return -1;
 844 #endif
 845           regs[i].set1(VMRegImpl::stack2reg(stk_args));
 846           stk_args += 2;
 847         }
 848         break;
 849       case T_LONG:
 850         assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 851         // fall through
 852       case T_OBJECT:
 853       case T_ARRAY:
 854       case T_ADDRESS:
 855       case T_METADATA:
 856         if (int_args < Argument::n_int_register_parameters_c) {
 857           regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
 858         } else {
 859           regs[i].set2(VMRegImpl::stack2reg(stk_args));
 860           stk_args += 2;
 861         }
 862         break;
 863       case T_FLOAT:
 864         if (fp_args < Argument::n_float_register_parameters_c) {
 865           regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
 866         } else {
 867 #ifdef __APPLE__
 868           // Less-than word types are stored one after another.
 869           // The code is unable to handle this so bailout.
 870           return -1;
 871 #endif
 872           regs[i].set1(VMRegImpl::stack2reg(stk_args));
 873           stk_args += 2;
 874         }
 875         break;
 876       case T_DOUBLE:
 877         assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 878         if (fp_args < Argument::n_float_register_parameters_c) {
 879           regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 880         } else {
 881           regs[i].set2(VMRegImpl::stack2reg(stk_args));
 882           stk_args += 2;
 883         }
 884         break;
 885       case T_VOID: // Halves of longs and doubles
 886         assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 887         regs[i].set_bad();
 888         break;
 889       default:
 890         ShouldNotReachHere();
 891         break;
 892       }
 893     }
 894 
 895   return stk_args;
 896 }
 897 
 898 int SharedRuntime::vector_calling_convention(VMRegPair *regs,
 899                                              uint num_bits,
 900                                              uint total_args_passed) {
 901   Unimplemented();
 902   return 0;
 903 }
 904 
 905 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
 906                                          VMRegPair *regs,
 907                                          VMRegPair *regs2,
 908                                          int total_args_passed)
 909 {
 910   int result = c_calling_convention_priv(sig_bt, regs, regs2, total_args_passed);
 911   guarantee(result >= 0, "Unsupported arguments configuration");
 912   return result;
 913 }
 914 
 915 
 916 void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
 917   // We always ignore the frame_slots arg and just use the space just below frame pointer
 918   // which by this time is free to use
 919   switch (ret_type) {
 920   case T_FLOAT:
 921     __ strs(v0, Address(rfp, -wordSize));
 922     break;
 923   case T_DOUBLE:
 924     __ strd(v0, Address(rfp, -wordSize));
 925     break;
 926   case T_VOID:  break;
 927   default: {
 928     __ str(r0, Address(rfp, -wordSize));
 929     }
 930   }
 931 }
 932 
 933 void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
 934   // We always ignore the frame_slots arg and just use the space just below frame pointer
 935   // which by this time is free to use
 936   switch (ret_type) {
 937   case T_FLOAT:
 938     __ ldrs(v0, Address(rfp, -wordSize));
 939     break;
 940   case T_DOUBLE:
 941     __ ldrd(v0, Address(rfp, -wordSize));
 942     break;
 943   case T_VOID:  break;
 944   default: {
 945     __ ldr(r0, Address(rfp, -wordSize));
 946     }
 947   }
 948 }
 949 static void save_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
 950   RegSet x;
 951   for ( int i = first_arg ; i < arg_count ; i++ ) {
 952     if (args[i].first()->is_Register()) {
 953       x = x + args[i].first()->as_Register();
 954     } else if (args[i].first()->is_FloatRegister()) {
 955       __ strd(args[i].first()->as_FloatRegister(), Address(__ pre(sp, -2 * wordSize)));
 956     }
 957   }
 958   __ push(x, sp);
 959 }
 960 
 961 static void restore_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
 962   RegSet x;
 963   for ( int i = first_arg ; i < arg_count ; i++ ) {
 964     if (args[i].first()->is_Register()) {
 965       x = x + args[i].first()->as_Register();
 966     } else {
 967       ;
 968     }
 969   }
 970   __ pop(x, sp);
 971   for ( int i = arg_count - 1 ; i >= first_arg ; i-- ) {
 972     if (args[i].first()->is_Register()) {
 973       ;
 974     } else if (args[i].first()->is_FloatRegister()) {
 975       __ ldrd(args[i].first()->as_FloatRegister(), Address(__ post(sp, 2 * wordSize)));
 976     }
 977   }
 978 }
 979 
 980 static void verify_oop_args(MacroAssembler* masm,
 981                             const methodHandle& method,
 982                             const BasicType* sig_bt,
 983                             const VMRegPair* regs) {
 984   Register temp_reg = r19;  // not part of any compiled calling seq
 985   if (VerifyOops) {
 986     for (int i = 0; i < method->size_of_parameters(); i++) {
 987       if (sig_bt[i] == T_OBJECT ||
 988           sig_bt[i] == T_ARRAY) {
 989         VMReg r = regs[i].first();
 990         assert(r->is_valid(), "bad oop arg");
 991         if (r->is_stack()) {
 992           __ ldr(temp_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size));
 993           __ verify_oop(temp_reg);
 994         } else {
 995           __ verify_oop(r->as_Register());
 996         }
 997       }
 998     }
 999   }
1000 }
1001 
1002 // defined in stubGenerator_aarch64.cpp
1003 OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots);
1004 void fill_continuation_entry(MacroAssembler* masm);
1005 void continuation_enter_cleanup(MacroAssembler* masm);
1006 
1007 // enterSpecial(Continuation c, boolean isContinue, boolean isVirtualThread)
1008 // On entry: c_rarg1 -- the continuation object
1009 //           c_rarg2 -- isContinue
1010 //           c_rarg3 -- isVirtualThread
1011 static void gen_continuation_enter(MacroAssembler* masm,
1012                                  const methodHandle& method,
1013                                  const BasicType* sig_bt,
1014                                  const VMRegPair* regs,
1015                                  int& exception_offset,
1016                                  OopMapSet*oop_maps,
1017                                  int& frame_complete,
1018                                  int& stack_slots,
1019                                  int& interpreted_entry_offset,
1020                                  int& compiled_entry_offset) {
1021   //verify_oop_args(masm, method, sig_bt, regs);
1022   Address resolve(SharedRuntime::get_resolve_static_call_stub(), relocInfo::static_call_type);
1023 
1024   address start = __ pc();
1025 
1026   Label call_thaw, exit;
1027 
1028   // i2i entry used at interp_only_mode only
1029   interpreted_entry_offset = __ pc() - start;
1030   {
1031 
1032 #ifdef ASSERT
1033     Label is_interp_only;
1034     __ ldrw(rscratch1, Address(rthread, JavaThread::interp_only_mode_offset()));
1035     __ cbnzw(rscratch1, is_interp_only);
1036     __ stop("enterSpecial interpreter entry called when not in interp_only_mode");
1037     __ bind(is_interp_only);
1038 #endif
1039 
1040     // Read interpreter arguments into registers (this is an ad-hoc i2c adapter)
1041     __ ldr(c_rarg1, Address(esp, Interpreter::stackElementSize*2));
1042     __ ldr(c_rarg2, Address(esp, Interpreter::stackElementSize*1));
1043     __ ldr(c_rarg3, Address(esp, Interpreter::stackElementSize*0));
1044     __ push_cont_fastpath(rthread);
1045 
1046     __ enter();
1047     stack_slots = 2; // will be adjusted in setup
1048     OopMap* map = continuation_enter_setup(masm, stack_slots);
1049     // The frame is complete here, but we only record it for the compiled entry, so the frame would appear unsafe,
1050     // but that's okay because at the very worst we'll miss an async sample, but we're in interp_only_mode anyway.
1051 
1052     fill_continuation_entry(masm);
1053 
1054     __ cbnz(c_rarg2, call_thaw);
1055 
1056     const address tr_call = __ trampoline_call(resolve);
1057 
1058     oop_maps->add_gc_map(__ pc() - start, map);
1059     __ post_call_nop();
1060 
1061     __ b(exit);
1062 
1063     CodeBuffer* cbuf = masm->code_section()->outer();
1064     CompiledStaticCall::emit_to_interp_stub(*cbuf, tr_call);
1065   }
1066 
1067   // compiled entry
1068   __ align(CodeEntryAlignment);
1069   compiled_entry_offset = __ pc() - start;
1070 
1071   __ enter();
1072   stack_slots = 2; // will be adjusted in setup
1073   OopMap* map = continuation_enter_setup(masm, stack_slots);
1074   frame_complete = __ pc() - start;
1075 
1076   fill_continuation_entry(masm);
1077 
1078   __ cbnz(c_rarg2, call_thaw);
1079 
1080   const address tr_call = __ trampoline_call(resolve);
1081 
1082   oop_maps->add_gc_map(__ pc() - start, map);
1083   __ post_call_nop();
1084 
1085   __ b(exit);
1086 
1087   __ bind(call_thaw);
1088 
1089   __ rt_call(CAST_FROM_FN_PTR(address, StubRoutines::cont_thaw()));
1090   oop_maps->add_gc_map(__ pc() - start, map->deep_copy());
1091   ContinuationEntry::_return_pc_offset = __ pc() - start;
1092   __ post_call_nop();
1093 
1094   __ bind(exit);
1095   continuation_enter_cleanup(masm);
1096   __ leave();
1097   __ ret(lr);
1098 
1099   /// exception handling
1100 
1101   exception_offset = __ pc() - start;
1102   {
1103       __ mov(r19, r0); // save return value contaning the exception oop in callee-saved R19
1104 
1105       continuation_enter_cleanup(masm);
1106 
1107       __ ldr(c_rarg1, Address(rfp, wordSize)); // return address
1108       __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, c_rarg1);
1109 
1110       // see OptoRuntime::generate_exception_blob: r0 -- exception oop, r3 -- exception pc
1111 
1112       __ mov(r1, r0); // the exception handler
1113       __ mov(r0, r19); // restore return value contaning the exception oop
1114       __ verify_oop(r0);
1115 
1116       __ leave();
1117       __ mov(r3, lr);
1118       __ br(r1); // the exception handler
1119   }
1120 
1121   CodeBuffer* cbuf = masm->code_section()->outer();
1122   CompiledStaticCall::emit_to_interp_stub(*cbuf, tr_call);
1123 }
1124 
1125 static void gen_continuation_yield(MacroAssembler* masm,
1126                                    const methodHandle& method,
1127                                    const BasicType* sig_bt,
1128                                    const VMRegPair* regs,
1129                                    OopMapSet* oop_maps,
1130                                    int& frame_complete,
1131                                    int& stack_slots,
1132                                    int& compiled_entry_offset) {
1133     enum layout {
1134       rfp_off1,
1135       rfp_off2,
1136       lr_off,
1137       lr_off2,
1138       framesize // inclusive of return address
1139     };
1140     // assert(is_even(framesize/2), "sp not 16-byte aligned");
1141     stack_slots = framesize /  VMRegImpl::slots_per_word;
1142     assert(stack_slots == 2, "recheck layout");
1143 
1144     address start = __ pc();
1145 
1146     compiled_entry_offset = __ pc() - start;
1147     __ enter();
1148 
1149     __ mov(c_rarg1, sp);
1150 
1151     frame_complete = __ pc() - start;
1152     address the_pc = __ pc();
1153 
1154     __ post_call_nop(); // this must be exactly after the pc value that is pushed into the frame info, we use this nop for fast CodeBlob lookup
1155 
1156     __ mov(c_rarg0, rthread);
1157     __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
1158     __ call_VM_leaf(Continuation::freeze_entry(), 2);
1159     __ reset_last_Java_frame(true);
1160 
1161     Label pinned;
1162 
1163     __ cbnz(r0, pinned);
1164 
1165     // We've succeeded, set sp to the ContinuationEntry
1166     __ ldr(rscratch1, Address(rthread, JavaThread::cont_entry_offset()));
1167     __ mov(sp, rscratch1);
1168     continuation_enter_cleanup(masm);
1169 
1170     __ bind(pinned); // pinned -- return to caller
1171 
1172     __ leave();
1173     __ ret(lr);
1174 
1175     OopMap* map = new OopMap(framesize, 1);
1176     oop_maps->add_gc_map(the_pc - start, map);
1177 }
1178 
1179 static void gen_special_dispatch(MacroAssembler* masm,
1180                                  const methodHandle& method,
1181                                  const BasicType* sig_bt,
1182                                  const VMRegPair* regs) {
1183   verify_oop_args(masm, method, sig_bt, regs);
1184   vmIntrinsics::ID iid = method->intrinsic_id();
1185 
1186   // Now write the args into the outgoing interpreter space
1187   bool     has_receiver   = false;
1188   Register receiver_reg   = noreg;
1189   int      member_arg_pos = -1;
1190   Register member_reg     = noreg;
1191   int      ref_kind       = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
1192   if (ref_kind != 0) {
1193     member_arg_pos = method->size_of_parameters() - 1;  // trailing MemberName argument
1194     member_reg = r19;  // known to be free at this point
1195     has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
1196   } else if (iid == vmIntrinsics::_invokeBasic) {
1197     has_receiver = true;
1198   } else if (iid == vmIntrinsics::_linkToNative) {
1199     member_arg_pos = method->size_of_parameters() - 1;  // trailing NativeEntryPoint argument
1200     member_reg = r19;  // known to be free at this point
1201   } else {
1202     fatal("unexpected intrinsic id %d", vmIntrinsics::as_int(iid));
1203   }
1204 
1205   if (member_reg != noreg) {
1206     // Load the member_arg into register, if necessary.
1207     SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
1208     VMReg r = regs[member_arg_pos].first();
1209     if (r->is_stack()) {
1210       __ ldr(member_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size));
1211     } else {
1212       // no data motion is needed
1213       member_reg = r->as_Register();
1214     }
1215   }
1216 
1217   if (has_receiver) {
1218     // Make sure the receiver is loaded into a register.
1219     assert(method->size_of_parameters() > 0, "oob");
1220     assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
1221     VMReg r = regs[0].first();
1222     assert(r->is_valid(), "bad receiver arg");
1223     if (r->is_stack()) {
1224       // Porting note:  This assumes that compiled calling conventions always
1225       // pass the receiver oop in a register.  If this is not true on some
1226       // platform, pick a temp and load the receiver from stack.
1227       fatal("receiver always in a register");
1228       receiver_reg = r2;  // known to be free at this point
1229       __ ldr(receiver_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size));
1230     } else {
1231       // no data motion is needed
1232       receiver_reg = r->as_Register();
1233     }
1234   }
1235 
1236   // Figure out which address we are really jumping to:
1237   MethodHandles::generate_method_handle_dispatch(masm, iid,
1238                                                  receiver_reg, member_reg, /*for_compiler_entry:*/ true);
1239 }
1240 
1241 // ---------------------------------------------------------------------------
1242 // Generate a native wrapper for a given method.  The method takes arguments
1243 // in the Java compiled code convention, marshals them to the native
1244 // convention (handlizes oops, etc), transitions to native, makes the call,
1245 // returns to java state (possibly blocking), unhandlizes any result and
1246 // returns.
1247 //
1248 // Critical native functions are a shorthand for the use of
1249 // GetPrimtiveArrayCritical and disallow the use of any other JNI
1250 // functions.  The wrapper is expected to unpack the arguments before
1251 // passing them to the callee. Critical native functions leave the state _in_Java,
1252 // since they block out GC.
1253 // Some other parts of JNI setup are skipped like the tear down of the JNI handle
1254 // block and the check for pending exceptions it's impossible for them
1255 // to be thrown.
1256 //
1257 nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
1258                                                 const methodHandle& method,
1259                                                 int compile_id,
1260                                                 BasicType* in_sig_bt,
1261                                                 VMRegPair* in_regs,
1262                                                 BasicType ret_type) {
1263   if (method->is_continuation_native_intrinsic()) {
1264     int exception_offset = -1;
1265     OopMapSet* oop_maps = new OopMapSet();
1266     int frame_complete = -1;
1267     int stack_slots = -1;
1268     int interpreted_entry_offset = -1;
1269     int vep_offset = -1;
1270     if (method->is_continuation_enter_intrinsic()) {
1271       gen_continuation_enter(masm,
1272                              method,
1273                              in_sig_bt,
1274                              in_regs,
1275                              exception_offset,
1276                              oop_maps,
1277                              frame_complete,
1278                              stack_slots,
1279                              interpreted_entry_offset,
1280                              vep_offset);
1281     } else if (method->is_continuation_yield_intrinsic()) {
1282       gen_continuation_yield(masm,
1283                              method,
1284                              in_sig_bt,
1285                              in_regs,
1286                              oop_maps,
1287                              frame_complete,
1288                              stack_slots,
1289                              vep_offset);
1290     } else {
1291       guarantee(false, "Unknown Continuation native intrinsic");
1292     }
1293 
1294 #ifdef ASSERT
1295     if (method->is_continuation_enter_intrinsic()) {
1296       assert(interpreted_entry_offset != -1, "Must be set");
1297       assert(exception_offset != -1,         "Must be set");
1298     } else {
1299       assert(interpreted_entry_offset == -1, "Must be unset");
1300       assert(exception_offset == -1,         "Must be unset");
1301     }
1302     assert(frame_complete != -1,    "Must be set");
1303     assert(stack_slots != -1,       "Must be set");
1304     assert(vep_offset != -1,        "Must be set");
1305 #endif
1306 
1307     __ flush();
1308     nmethod* nm = nmethod::new_native_nmethod(method,
1309                                               compile_id,
1310                                               masm->code(),
1311                                               vep_offset,
1312                                               frame_complete,
1313                                               stack_slots,
1314                                               in_ByteSize(-1),
1315                                               in_ByteSize(-1),
1316                                               oop_maps,
1317                                               exception_offset);
1318     if (method->is_continuation_enter_intrinsic()) {
1319       ContinuationEntry::set_enter_code(nm, interpreted_entry_offset);
1320     } else if (method->is_continuation_yield_intrinsic()) {
1321       _cont_doYield_stub = nm;
1322     } else {
1323       guarantee(false, "Unknown Continuation native intrinsic");
1324     }
1325     return nm;
1326   }
1327 
1328   if (method->is_method_handle_intrinsic()) {
1329     vmIntrinsics::ID iid = method->intrinsic_id();
1330     intptr_t start = (intptr_t)__ pc();
1331     int vep_offset = ((intptr_t)__ pc()) - start;
1332 
1333     // First instruction must be a nop as it may need to be patched on deoptimisation
1334     __ nop();
1335     gen_special_dispatch(masm,
1336                          method,
1337                          in_sig_bt,
1338                          in_regs);
1339     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
1340     __ flush();
1341     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
1342     return nmethod::new_native_nmethod(method,
1343                                        compile_id,
1344                                        masm->code(),
1345                                        vep_offset,
1346                                        frame_complete,
1347                                        stack_slots / VMRegImpl::slots_per_word,
1348                                        in_ByteSize(-1),
1349                                        in_ByteSize(-1),
1350                                        (OopMapSet*)NULL);
1351   }
1352   address native_func = method->native_function();
1353   assert(native_func != NULL, "must have function");
1354 
1355   // An OopMap for lock (and class if static)
1356   OopMapSet *oop_maps = new OopMapSet();
1357   intptr_t start = (intptr_t)__ pc();
1358 
1359   // We have received a description of where all the java arg are located
1360   // on entry to the wrapper. We need to convert these args to where
1361   // the jni function will expect them. To figure out where they go
1362   // we convert the java signature to a C signature by inserting
1363   // the hidden arguments as arg[0] and possibly arg[1] (static method)
1364 
1365   const int total_in_args = method->size_of_parameters();
1366   int total_c_args = total_in_args + (method->is_static() ? 2 : 1);
1367 
1368   BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
1369   VMRegPair* out_regs   = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
1370   BasicType* in_elem_bt = NULL;
1371 
1372   int argc = 0;
1373   out_sig_bt[argc++] = T_ADDRESS;
1374   if (method->is_static()) {
1375     out_sig_bt[argc++] = T_OBJECT;
1376   }
1377 
1378   for (int i = 0; i < total_in_args ; i++ ) {
1379     out_sig_bt[argc++] = in_sig_bt[i];
1380   }
1381 
1382   // Now figure out where the args must be stored and how much stack space
1383   // they require.
1384   int out_arg_slots;
1385   out_arg_slots = c_calling_convention_priv(out_sig_bt, out_regs, NULL, total_c_args);
1386 
1387   if (out_arg_slots < 0) {
1388     return NULL;
1389   }
1390 
1391   // Compute framesize for the wrapper.  We need to handlize all oops in
1392   // incoming registers
1393 
1394   // Calculate the total number of stack slots we will need.
1395 
1396   // First count the abi requirement plus all of the outgoing args
1397   int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots;
1398 
1399   // Now the space for the inbound oop handle area
1400   int total_save_slots = 8 * VMRegImpl::slots_per_word;  // 8 arguments passed in registers
1401 
1402   int oop_handle_offset = stack_slots;
1403   stack_slots += total_save_slots;
1404 
1405   // Now any space we need for handlizing a klass if static method
1406 
1407   int klass_slot_offset = 0;
1408   int klass_offset = -1;
1409   int lock_slot_offset = 0;
1410   bool is_static = false;
1411 
1412   if (method->is_static()) {
1413     klass_slot_offset = stack_slots;
1414     stack_slots += VMRegImpl::slots_per_word;
1415     klass_offset = klass_slot_offset * VMRegImpl::stack_slot_size;
1416     is_static = true;
1417   }
1418 
1419   // Plus a lock if needed
1420 
1421   if (method->is_synchronized()) {
1422     lock_slot_offset = stack_slots;
1423     stack_slots += VMRegImpl::slots_per_word;
1424   }
1425 
1426   // Now a place (+2) to save return values or temp during shuffling
1427   // + 4 for return address (which we own) and saved rfp
1428   stack_slots += 6;
1429 
1430   // Ok The space we have allocated will look like:
1431   //
1432   //
1433   // FP-> |                     |
1434   //      |---------------------|
1435   //      | 2 slots for moves   |
1436   //      |---------------------|
1437   //      | lock box (if sync)  |
1438   //      |---------------------| <- lock_slot_offset
1439   //      | klass (if static)   |
1440   //      |---------------------| <- klass_slot_offset
1441   //      | oopHandle area      |
1442   //      |---------------------| <- oop_handle_offset (8 java arg registers)
1443   //      | outbound memory     |
1444   //      | based arguments     |
1445   //      |                     |
1446   //      |---------------------|
1447   //      |                     |
1448   // SP-> | out_preserved_slots |
1449   //
1450   //
1451 
1452 
1453   // Now compute actual number of stack words we need rounding to make
1454   // stack properly aligned.
1455   stack_slots = align_up(stack_slots, StackAlignmentInSlots);
1456 
1457   int stack_size = stack_slots * VMRegImpl::stack_slot_size;
1458 
1459   // First thing make an ic check to see if we should even be here
1460 
1461   // We are free to use all registers as temps without saving them and
1462   // restoring them except rfp. rfp is the only callee save register
1463   // as far as the interpreter and the compiler(s) are concerned.
1464 
1465 
1466   const Register ic_reg = rscratch2;
1467   const Register receiver = j_rarg0;
1468 
1469   Label hit;
1470   Label exception_pending;
1471 
1472   assert_different_registers(ic_reg, receiver, rscratch1);
1473   __ verify_oop(receiver);
1474   __ cmp_klass(receiver, ic_reg, rscratch1);
1475   __ br(Assembler::EQ, hit);
1476 
1477   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1478 
1479   // Verified entry point must be aligned
1480   __ align(8);
1481 
1482   __ bind(hit);
1483 
1484   int vep_offset = ((intptr_t)__ pc()) - start;
1485 
1486   // If we have to make this method not-entrant we'll overwrite its
1487   // first instruction with a jump.  For this action to be legal we
1488   // must ensure that this first instruction is a B, BL, NOP, BKPT,
1489   // SVC, HVC, or SMC.  Make it a NOP.
1490   __ nop();
1491 
1492   if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
1493     Label L_skip_barrier;
1494     __ mov_metadata(rscratch2, method->method_holder()); // InstanceKlass*
1495     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1496     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1497 
1498     __ bind(L_skip_barrier);
1499   }
1500 
1501   // Generate stack overflow check
1502   __ bang_stack_with_offset(checked_cast<int>(StackOverflow::stack_shadow_zone_size()));
1503 
1504   // Generate a new frame for the wrapper.
1505   __ enter();
1506   // -2 because return address is already present and so is saved rfp
1507   __ sub(sp, sp, stack_size - 2*wordSize);
1508 
1509   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1510   bs->nmethod_entry_barrier(masm, NULL /* slow_path */, NULL /* continuation */, NULL /* guard */);
1511 
1512   // Frame is now completed as far as size and linkage.
1513   int frame_complete = ((intptr_t)__ pc()) - start;
1514 
1515   // We use r20 as the oop handle for the receiver/klass
1516   // It is callee save so it survives the call to native
1517 
1518   const Register oop_handle_reg = r20;
1519 
1520   //
1521   // We immediately shuffle the arguments so that any vm call we have to
1522   // make from here on out (sync slow path, jvmti, etc.) we will have
1523   // captured the oops from our caller and have a valid oopMap for
1524   // them.
1525 
1526   // -----------------
1527   // The Grand Shuffle
1528 
1529   // The Java calling convention is either equal (linux) or denser (win64) than the
1530   // c calling convention. However the because of the jni_env argument the c calling
1531   // convention always has at least one more (and two for static) arguments than Java.
1532   // Therefore if we move the args from java -> c backwards then we will never have
1533   // a register->register conflict and we don't have to build a dependency graph
1534   // and figure out how to break any cycles.
1535   //
1536 
1537   // Record esp-based slot for receiver on stack for non-static methods
1538   int receiver_offset = -1;
1539 
1540   // This is a trick. We double the stack slots so we can claim
1541   // the oops in the caller's frame. Since we are sure to have
1542   // more args than the caller doubling is enough to make
1543   // sure we can capture all the incoming oop args from the
1544   // caller.
1545   //
1546   OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1547 
1548   // Mark location of rfp (someday)
1549   // map->set_callee_saved(VMRegImpl::stack2reg( stack_slots - 2), stack_slots * 2, 0, vmreg(rfp));
1550 
1551 
1552   int float_args = 0;
1553   int int_args = 0;
1554 
1555 #ifdef ASSERT
1556   bool reg_destroyed[Register::number_of_registers];
1557   bool freg_destroyed[FloatRegister::number_of_registers];
1558   for ( int r = 0 ; r < Register::number_of_registers ; r++ ) {
1559     reg_destroyed[r] = false;
1560   }
1561   for ( int f = 0 ; f < FloatRegister::number_of_registers ; f++ ) {
1562     freg_destroyed[f] = false;
1563   }
1564 
1565 #endif /* ASSERT */
1566 
1567   // For JNI natives the incoming and outgoing registers are offset upwards.
1568   GrowableArray<int> arg_order(2 * total_in_args);
1569   VMRegPair tmp_vmreg;
1570   tmp_vmreg.set2(r19->as_VMReg());
1571 
1572   for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
1573     arg_order.push(i);
1574     arg_order.push(c_arg);
1575   }
1576 
1577   int temploc = -1;
1578   for (int ai = 0; ai < arg_order.length(); ai += 2) {
1579     int i = arg_order.at(ai);
1580     int c_arg = arg_order.at(ai + 1);
1581     __ block_comment(err_msg("move %d -> %d", i, c_arg));
1582     assert(c_arg != -1 && i != -1, "wrong order");
1583 #ifdef ASSERT
1584     if (in_regs[i].first()->is_Register()) {
1585       assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
1586     } else if (in_regs[i].first()->is_FloatRegister()) {
1587       assert(!freg_destroyed[in_regs[i].first()->as_FloatRegister()->encoding()], "destroyed reg!");
1588     }
1589     if (out_regs[c_arg].first()->is_Register()) {
1590       reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1591     } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1592       freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1593     }
1594 #endif /* ASSERT */
1595     switch (in_sig_bt[i]) {
1596       case T_ARRAY:
1597       case T_OBJECT:
1598         __ object_move(map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
1599                        ((i == 0) && (!is_static)),
1600                        &receiver_offset);
1601         int_args++;
1602         break;
1603       case T_VOID:
1604         break;
1605 
1606       case T_FLOAT:
1607         __ float_move(in_regs[i], out_regs[c_arg]);
1608         float_args++;
1609         break;
1610 
1611       case T_DOUBLE:
1612         assert( i + 1 < total_in_args &&
1613                 in_sig_bt[i + 1] == T_VOID &&
1614                 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
1615         __ double_move(in_regs[i], out_regs[c_arg]);
1616         float_args++;
1617         break;
1618 
1619       case T_LONG :
1620         __ long_move(in_regs[i], out_regs[c_arg]);
1621         int_args++;
1622         break;
1623 
1624       case T_ADDRESS: assert(false, "found T_ADDRESS in java args");
1625 
1626       default:
1627         __ move32_64(in_regs[i], out_regs[c_arg]);
1628         int_args++;
1629     }
1630   }
1631 
1632   // point c_arg at the first arg that is already loaded in case we
1633   // need to spill before we call out
1634   int c_arg = total_c_args - total_in_args;
1635 
1636   // Pre-load a static method's oop into c_rarg1.
1637   if (method->is_static()) {
1638 
1639     //  load oop into a register
1640     __ movoop(c_rarg1,
1641               JNIHandles::make_local(method->method_holder()->java_mirror()));
1642 
1643     // Now handlize the static class mirror it's known not-null.
1644     __ str(c_rarg1, Address(sp, klass_offset));
1645     map->set_oop(VMRegImpl::stack2reg(klass_slot_offset));
1646 
1647     // Now get the handle
1648     __ lea(c_rarg1, Address(sp, klass_offset));
1649     // and protect the arg if we must spill
1650     c_arg--;
1651   }
1652 
1653   // Change state to native (we save the return address in the thread, since it might not
1654   // be pushed on the stack when we do a stack traversal).
1655   // We use the same pc/oopMap repeatedly when we call out
1656 
1657   Label native_return;
1658   __ set_last_Java_frame(sp, noreg, native_return, rscratch1);
1659 
1660   Label dtrace_method_entry, dtrace_method_entry_done;
1661   {
1662     uint64_t offset;
1663     __ adrp(rscratch1, ExternalAddress((address)&DTraceMethodProbes), offset);
1664     __ ldrb(rscratch1, Address(rscratch1, offset));
1665     __ cbnzw(rscratch1, dtrace_method_entry);
1666     __ bind(dtrace_method_entry_done);
1667   }
1668 
1669   // RedefineClasses() tracing support for obsolete method entry
1670   if (log_is_enabled(Trace, redefine, class, obsolete)) {
1671     // protect the args we've loaded
1672     save_args(masm, total_c_args, c_arg, out_regs);
1673     __ mov_metadata(c_rarg1, method());
1674     __ call_VM_leaf(
1675       CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
1676       rthread, c_rarg1);
1677     restore_args(masm, total_c_args, c_arg, out_regs);
1678   }
1679 
1680   // Lock a synchronized method
1681 
1682   // Register definitions used by locking and unlocking
1683 
1684   const Register swap_reg = r0;
1685   const Register obj_reg  = r19;  // Will contain the oop
1686   const Register lock_reg = r13;  // Address of compiler lock object (BasicLock)
1687   const Register old_hdr  = r13;  // value of old header at unlock time
1688   const Register tmp = lr;
1689 
1690   Label slow_path_lock;
1691   Label lock_done;
1692 
1693   if (method->is_synchronized()) {
1694     Label count;
1695     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
1696 
1697     // Get the handle (the 2nd argument)
1698     __ mov(oop_handle_reg, c_rarg1);
1699 
1700     // Get address of the box
1701 
1702     __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1703 
1704     // Load the oop from the handle
1705     __ ldr(obj_reg, Address(oop_handle_reg, 0));
1706 
1707     if (!UseHeavyMonitors) {
1708       // Load (object->mark() | 1) into swap_reg %r0
1709       __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
1710       __ orr(swap_reg, rscratch1, 1);
1711 
1712       // Save (object->mark() | 1) into BasicLock's displaced header
1713       __ str(swap_reg, Address(lock_reg, mark_word_offset));
1714 
1715       // src -> dest iff dest == r0 else r0 <- dest
1716       __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, count, /*fallthrough*/NULL);
1717 
1718       // Hmm should this move to the slow path code area???
1719 
1720       // Test if the oopMark is an obvious stack pointer, i.e.,
1721       //  1) (mark & 3) == 0, and
1722       //  2) sp <= mark < mark + os::pagesize()
1723       // These 3 tests can be done by evaluating the following
1724       // expression: ((mark - sp) & (3 - os::vm_page_size())),
1725       // assuming both stack pointer and pagesize have their
1726       // least significant 2 bits clear.
1727       // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
1728 
1729       __ sub(swap_reg, sp, swap_reg);
1730       __ neg(swap_reg, swap_reg);
1731       __ ands(swap_reg, swap_reg, 3 - os::vm_page_size());
1732 
1733       // Save the test result, for recursive case, the result is zero
1734       __ str(swap_reg, Address(lock_reg, mark_word_offset));
1735       __ br(Assembler::NE, slow_path_lock);
1736     } else {
1737       __ b(slow_path_lock);
1738     }
1739     __ bind(count);
1740     __ increment(Address(rthread, JavaThread::held_monitor_count_offset()));
1741 
1742     // Slow path will re-enter here
1743     __ bind(lock_done);
1744   }
1745 
1746 
1747   // Finally just about ready to make the JNI call
1748 
1749   // get JNIEnv* which is first argument to native
1750   __ lea(c_rarg0, Address(rthread, in_bytes(JavaThread::jni_environment_offset())));
1751 
1752   // Now set thread in native
1753   __ mov(rscratch1, _thread_in_native);
1754   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
1755   __ stlrw(rscratch1, rscratch2);
1756 
1757   __ rt_call(native_func);
1758 
1759   __ bind(native_return);
1760 
1761   intptr_t return_pc = (intptr_t) __ pc();
1762   oop_maps->add_gc_map(return_pc - start, map);
1763 
1764   // Unpack native results.
1765   switch (ret_type) {
1766   case T_BOOLEAN: __ c2bool(r0);                     break;
1767   case T_CHAR   : __ ubfx(r0, r0, 0, 16);            break;
1768   case T_BYTE   : __ sbfx(r0, r0, 0, 8);             break;
1769   case T_SHORT  : __ sbfx(r0, r0, 0, 16);            break;
1770   case T_INT    : __ sbfx(r0, r0, 0, 32);            break;
1771   case T_DOUBLE :
1772   case T_FLOAT  :
1773     // Result is in v0 we'll save as needed
1774     break;
1775   case T_ARRAY:                 // Really a handle
1776   case T_OBJECT:                // Really a handle
1777       break; // can't de-handlize until after safepoint check
1778   case T_VOID: break;
1779   case T_LONG: break;
1780   default       : ShouldNotReachHere();
1781   }
1782 
1783   Label safepoint_in_progress, safepoint_in_progress_done;
1784   Label after_transition;
1785 
1786   // Switch thread to "native transition" state before reading the synchronization state.
1787   // This additional state is necessary because reading and testing the synchronization
1788   // state is not atomic w.r.t. GC, as this scenario demonstrates:
1789   //     Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
1790   //     VM thread changes sync state to synchronizing and suspends threads for GC.
1791   //     Thread A is resumed to finish this native method, but doesn't block here since it
1792   //     didn't see any synchronization is progress, and escapes.
1793   __ mov(rscratch1, _thread_in_native_trans);
1794 
1795   __ strw(rscratch1, Address(rthread, JavaThread::thread_state_offset()));
1796 
1797   // Force this write out before the read below
1798   if (!UseSystemMemoryBarrier) {
1799     __ dmb(Assembler::ISH);
1800   }
1801 
1802   __ verify_sve_vector_length();
1803 
1804   // Check for safepoint operation in progress and/or pending suspend requests.
1805   {
1806     // We need an acquire here to ensure that any subsequent load of the
1807     // global SafepointSynchronize::_state flag is ordered after this load
1808     // of the thread-local polling word.  We don't want this poll to
1809     // return false (i.e. not safepointing) and a later poll of the global
1810     // SafepointSynchronize::_state spuriously to return true.
1811     //
1812     // This is to avoid a race when we're in a native->Java transition
1813     // racing the code which wakes up from a safepoint.
1814 
1815     __ safepoint_poll(safepoint_in_progress, true /* at_return */, true /* acquire */, false /* in_nmethod */);
1816     __ ldrw(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
1817     __ cbnzw(rscratch1, safepoint_in_progress);
1818     __ bind(safepoint_in_progress_done);
1819   }
1820 
1821   // change thread state
1822   __ mov(rscratch1, _thread_in_Java);
1823   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
1824   __ stlrw(rscratch1, rscratch2);
1825   __ bind(after_transition);
1826 
1827   Label reguard;
1828   Label reguard_done;
1829   __ ldrb(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
1830   __ cmpw(rscratch1, StackOverflow::stack_guard_yellow_reserved_disabled);
1831   __ br(Assembler::EQ, reguard);
1832   __ bind(reguard_done);
1833 
1834   // native result if any is live
1835 
1836   // Unlock
1837   Label unlock_done;
1838   Label slow_path_unlock;
1839   if (method->is_synchronized()) {
1840 
1841     // Get locked oop from the handle we passed to jni
1842     __ ldr(obj_reg, Address(oop_handle_reg, 0));
1843 
1844     Label done, not_recursive;
1845 
1846     if (!UseHeavyMonitors) {
1847       // Simple recursive lock?
1848       __ ldr(rscratch1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1849       __ cbnz(rscratch1, not_recursive);
1850       __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
1851       __ b(done);
1852     }
1853 
1854     __ bind(not_recursive);
1855 
1856     // Must save r0 if if it is live now because cmpxchg must use it
1857     if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
1858       save_native_result(masm, ret_type, stack_slots);
1859     }
1860 
1861     if (!UseHeavyMonitors) {
1862       // get address of the stack lock
1863       __ lea(r0, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1864       //  get old displaced header
1865       __ ldr(old_hdr, Address(r0, 0));
1866 
1867       // Atomic swap old header if oop still contains the stack lock
1868       Label count;
1869       __ cmpxchg_obj_header(r0, old_hdr, obj_reg, rscratch1, count, &slow_path_unlock);
1870       __ bind(count);
1871       __ decrement(Address(rthread, JavaThread::held_monitor_count_offset()));
1872     } else {
1873       __ b(slow_path_unlock);
1874     }
1875 
1876     // slow path re-enters here
1877     __ bind(unlock_done);
1878     if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
1879       restore_native_result(masm, ret_type, stack_slots);
1880     }
1881 
1882     __ bind(done);
1883   }
1884 
1885   Label dtrace_method_exit, dtrace_method_exit_done;
1886   {
1887     uint64_t offset;
1888     __ adrp(rscratch1, ExternalAddress((address)&DTraceMethodProbes), offset);
1889     __ ldrb(rscratch1, Address(rscratch1, offset));
1890     __ cbnzw(rscratch1, dtrace_method_exit);
1891     __ bind(dtrace_method_exit_done);
1892   }
1893 
1894   __ reset_last_Java_frame(false);
1895 
1896   // Unbox oop result, e.g. JNIHandles::resolve result.
1897   if (is_reference_type(ret_type)) {
1898     __ resolve_jobject(r0, r1, r2);
1899   }
1900 
1901   if (CheckJNICalls) {
1902     // clear_pending_jni_exception_check
1903     __ str(zr, Address(rthread, JavaThread::pending_jni_exception_check_fn_offset()));
1904   }
1905 
1906   // reset handle block
1907   __ ldr(r2, Address(rthread, JavaThread::active_handles_offset()));
1908   __ str(zr, Address(r2, JNIHandleBlock::top_offset_in_bytes()));
1909 
1910   __ leave();
1911 
1912   // Any exception pending?
1913   __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1914   __ cbnz(rscratch1, exception_pending);
1915 
1916   // We're done
1917   __ ret(lr);
1918 
1919   // Unexpected paths are out of line and go here
1920 
1921   // forward the exception
1922   __ bind(exception_pending);
1923 
1924   // and forward the exception
1925   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
1926 
1927   // Slow path locking & unlocking
1928   if (method->is_synchronized()) {
1929 
1930     __ block_comment("Slow path lock {");
1931     __ bind(slow_path_lock);
1932 
1933     // has last_Java_frame setup. No exceptions so do vanilla call not call_VM
1934     // args are (oop obj, BasicLock* lock, JavaThread* thread)
1935 
1936     // protect the args we've loaded
1937     save_args(masm, total_c_args, c_arg, out_regs);
1938 
1939     __ mov(c_rarg0, obj_reg);
1940     __ mov(c_rarg1, lock_reg);
1941     __ mov(c_rarg2, rthread);
1942 
1943     // Not a leaf but we have last_Java_frame setup as we want
1944     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3);
1945     restore_args(masm, total_c_args, c_arg, out_regs);
1946 
1947 #ifdef ASSERT
1948     { Label L;
1949       __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1950       __ cbz(rscratch1, L);
1951       __ stop("no pending exception allowed on exit from monitorenter");
1952       __ bind(L);
1953     }
1954 #endif
1955     __ b(lock_done);
1956 
1957     __ block_comment("} Slow path lock");
1958 
1959     __ block_comment("Slow path unlock {");
1960     __ bind(slow_path_unlock);
1961 
1962     // If we haven't already saved the native result we must save it now as xmm registers
1963     // are still exposed.
1964 
1965     if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
1966       save_native_result(masm, ret_type, stack_slots);
1967     }
1968 
1969     __ mov(c_rarg2, rthread);
1970     __ lea(c_rarg1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
1971     __ mov(c_rarg0, obj_reg);
1972 
1973     // Save pending exception around call to VM (which contains an EXCEPTION_MARK)
1974     // NOTE that obj_reg == r19 currently
1975     __ ldr(r19, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1976     __ str(zr, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1977 
1978     __ rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C));
1979 
1980 #ifdef ASSERT
1981     {
1982       Label L;
1983       __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1984       __ cbz(rscratch1, L);
1985       __ stop("no pending exception allowed on exit complete_monitor_unlocking_C");
1986       __ bind(L);
1987     }
1988 #endif /* ASSERT */
1989 
1990     __ str(r19, Address(rthread, in_bytes(Thread::pending_exception_offset())));
1991 
1992     if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
1993       restore_native_result(masm, ret_type, stack_slots);
1994     }
1995     __ b(unlock_done);
1996 
1997     __ block_comment("} Slow path unlock");
1998 
1999   } // synchronized
2000 
2001   // SLOW PATH Reguard the stack if needed
2002 
2003   __ bind(reguard);
2004   save_native_result(masm, ret_type, stack_slots);
2005   __ rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
2006   restore_native_result(masm, ret_type, stack_slots);
2007   // and continue
2008   __ b(reguard_done);
2009 
2010   // SLOW PATH safepoint
2011   {
2012     __ block_comment("safepoint {");
2013     __ bind(safepoint_in_progress);
2014 
2015     // Don't use call_VM as it will see a possible pending exception and forward it
2016     // and never return here preventing us from clearing _last_native_pc down below.
2017     //
2018     save_native_result(masm, ret_type, stack_slots);
2019     __ mov(c_rarg0, rthread);
2020 #ifndef PRODUCT
2021   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
2022 #endif
2023     __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
2024     __ blr(rscratch1);
2025 
2026     // Restore any method result value
2027     restore_native_result(masm, ret_type, stack_slots);
2028 
2029     __ b(safepoint_in_progress_done);
2030     __ block_comment("} safepoint");
2031   }
2032 
2033   // SLOW PATH dtrace support
2034   {
2035     __ block_comment("dtrace entry {");
2036     __ bind(dtrace_method_entry);
2037 
2038     // We have all of the arguments setup at this point. We must not touch any register
2039     // argument registers at this point (what if we save/restore them there are no oop?
2040 
2041     save_args(masm, total_c_args, c_arg, out_regs);
2042     __ mov_metadata(c_rarg1, method());
2043     __ call_VM_leaf(
2044       CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
2045       rthread, c_rarg1);
2046     restore_args(masm, total_c_args, c_arg, out_regs);
2047     __ b(dtrace_method_entry_done);
2048     __ block_comment("} dtrace entry");
2049   }
2050 
2051   {
2052     __ block_comment("dtrace exit {");
2053     __ bind(dtrace_method_exit);
2054     save_native_result(masm, ret_type, stack_slots);
2055     __ mov_metadata(c_rarg1, method());
2056     __ call_VM_leaf(
2057          CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2058          rthread, c_rarg1);
2059     restore_native_result(masm, ret_type, stack_slots);
2060     __ b(dtrace_method_exit_done);
2061     __ block_comment("} dtrace exit");
2062   }
2063 
2064 
2065   __ flush();
2066 
2067   nmethod *nm = nmethod::new_native_nmethod(method,
2068                                             compile_id,
2069                                             masm->code(),
2070                                             vep_offset,
2071                                             frame_complete,
2072                                             stack_slots / VMRegImpl::slots_per_word,
2073                                             (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)),
2074                                             in_ByteSize(lock_slot_offset*VMRegImpl::stack_slot_size),
2075                                             oop_maps);
2076 
2077   return nm;
2078 }
2079 
2080 // this function returns the adjust size (in number of words) to a c2i adapter
2081 // activation for use during deoptimization
2082 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals) {
2083   assert(callee_locals >= callee_parameters,
2084           "test and remove; got more parms than locals");
2085   if (callee_locals < callee_parameters)
2086     return 0;                   // No adjustment for negative locals
2087   int diff = (callee_locals - callee_parameters) * Interpreter::stackElementWords;
2088   // diff is counted in stack words
2089   return align_up(diff, 2);
2090 }
2091 
2092 
2093 //------------------------------generate_deopt_blob----------------------------
2094 void SharedRuntime::generate_deopt_blob() {
2095   // Allocate space for the code
2096   ResourceMark rm;
2097   // Setup code generation tools
2098   int pad = 0;
2099 #if INCLUDE_JVMCI
2100   if (EnableJVMCI) {
2101     pad += 512; // Increase the buffer size when compiling for JVMCI
2102   }
2103 #endif
2104   CodeBuffer buffer("deopt_blob", 2048+pad, 1024);
2105   MacroAssembler* masm = new MacroAssembler(&buffer);
2106   int frame_size_in_words;
2107   OopMap* map = NULL;
2108   OopMapSet *oop_maps = new OopMapSet();
2109   RegisterSaver reg_save(COMPILER2_OR_JVMCI != 0);
2110 
2111   // -------------
2112   // This code enters when returning to a de-optimized nmethod.  A return
2113   // address has been pushed on the stack, and return values are in
2114   // registers.
2115   // If we are doing a normal deopt then we were called from the patched
2116   // nmethod from the point we returned to the nmethod. So the return
2117   // address on the stack is wrong by NativeCall::instruction_size
2118   // We will adjust the value so it looks like we have the original return
2119   // address on the stack (like when we eagerly deoptimized).
2120   // In the case of an exception pending when deoptimizing, we enter
2121   // with a return address on the stack that points after the call we patched
2122   // into the exception handler. We have the following register state from,
2123   // e.g., the forward exception stub (see stubGenerator_x86_64.cpp).
2124   //    r0: exception oop
2125   //    r19: exception handler
2126   //    r3: throwing pc
2127   // So in this case we simply jam r3 into the useless return address and
2128   // the stack looks just like we want.
2129   //
2130   // At this point we need to de-opt.  We save the argument return
2131   // registers.  We call the first C routine, fetch_unroll_info().  This
2132   // routine captures the return values and returns a structure which
2133   // describes the current frame size and the sizes of all replacement frames.
2134   // The current frame is compiled code and may contain many inlined
2135   // functions, each with their own JVM state.  We pop the current frame, then
2136   // push all the new frames.  Then we call the C routine unpack_frames() to
2137   // populate these frames.  Finally unpack_frames() returns us the new target
2138   // address.  Notice that callee-save registers are BLOWN here; they have
2139   // already been captured in the vframeArray at the time the return PC was
2140   // patched.
2141   address start = __ pc();
2142   Label cont;
2143 
2144   // Prolog for non exception case!
2145 
2146   // Save everything in sight.
2147   map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2148 
2149   // Normal deoptimization.  Save exec mode for unpack_frames.
2150   __ movw(rcpool, Deoptimization::Unpack_deopt); // callee-saved
2151   __ b(cont);
2152 
2153   int reexecute_offset = __ pc() - start;
2154 #if INCLUDE_JVMCI && !defined(COMPILER1)
2155   if (EnableJVMCI && UseJVMCICompiler) {
2156     // JVMCI does not use this kind of deoptimization
2157     __ should_not_reach_here();
2158   }
2159 #endif
2160 
2161   // Reexecute case
2162   // return address is the pc describes what bci to do re-execute at
2163 
2164   // No need to update map as each call to save_live_registers will produce identical oopmap
2165   (void) reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2166 
2167   __ movw(rcpool, Deoptimization::Unpack_reexecute); // callee-saved
2168   __ b(cont);
2169 
2170 #if INCLUDE_JVMCI
2171   Label after_fetch_unroll_info_call;
2172   int implicit_exception_uncommon_trap_offset = 0;
2173   int uncommon_trap_offset = 0;
2174 
2175   if (EnableJVMCI) {
2176     implicit_exception_uncommon_trap_offset = __ pc() - start;
2177 
2178     __ ldr(lr, Address(rthread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())));
2179     __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())));
2180 
2181     uncommon_trap_offset = __ pc() - start;
2182 
2183     // Save everything in sight.
2184     reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2185     // fetch_unroll_info needs to call last_java_frame()
2186     Label retaddr;
2187     __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2188 
2189     __ ldrw(c_rarg1, Address(rthread, in_bytes(JavaThread::pending_deoptimization_offset())));
2190     __ movw(rscratch1, -1);
2191     __ strw(rscratch1, Address(rthread, in_bytes(JavaThread::pending_deoptimization_offset())));
2192 
2193     __ movw(rcpool, (int32_t)Deoptimization::Unpack_reexecute);
2194     __ mov(c_rarg0, rthread);
2195     __ movw(c_rarg2, rcpool); // exec mode
2196     __ lea(rscratch1,
2197            RuntimeAddress(CAST_FROM_FN_PTR(address,
2198                                            Deoptimization::uncommon_trap)));
2199     __ blr(rscratch1);
2200     __ bind(retaddr);
2201     oop_maps->add_gc_map( __ pc()-start, map->deep_copy());
2202 
2203     __ reset_last_Java_frame(false);
2204 
2205     __ b(after_fetch_unroll_info_call);
2206   } // EnableJVMCI
2207 #endif // INCLUDE_JVMCI
2208 
2209   int exception_offset = __ pc() - start;
2210 
2211   // Prolog for exception case
2212 
2213   // all registers are dead at this entry point, except for r0, and
2214   // r3 which contain the exception oop and exception pc
2215   // respectively.  Set them in TLS and fall thru to the
2216   // unpack_with_exception_in_tls entry point.
2217 
2218   __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
2219   __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
2220 
2221   int exception_in_tls_offset = __ pc() - start;
2222 
2223   // new implementation because exception oop is now passed in JavaThread
2224 
2225   // Prolog for exception case
2226   // All registers must be preserved because they might be used by LinearScan
2227   // Exceptiop oop and throwing PC are passed in JavaThread
2228   // tos: stack at point of call to method that threw the exception (i.e. only
2229   // args are on the stack, no return address)
2230 
2231   // The return address pushed by save_live_registers will be patched
2232   // later with the throwing pc. The correct value is not available
2233   // now because loading it from memory would destroy registers.
2234 
2235   // NB: The SP at this point must be the SP of the method that is
2236   // being deoptimized.  Deoptimization assumes that the frame created
2237   // here by save_live_registers is immediately below the method's SP.
2238   // This is a somewhat fragile mechanism.
2239 
2240   // Save everything in sight.
2241   map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2242 
2243   // Now it is safe to overwrite any register
2244 
2245   // Deopt during an exception.  Save exec mode for unpack_frames.
2246   __ mov(rcpool, Deoptimization::Unpack_exception); // callee-saved
2247 
2248   // load throwing pc from JavaThread and patch it as the return address
2249   // of the current frame. Then clear the field in JavaThread
2250   __ ldr(r3, Address(rthread, JavaThread::exception_pc_offset()));
2251   __ protect_return_address(r3, rscratch1);
2252   __ str(r3, Address(rfp, wordSize));
2253   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
2254 
2255 #ifdef ASSERT
2256   // verify that there is really an exception oop in JavaThread
2257   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
2258   __ verify_oop(r0);
2259 
2260   // verify that there is no pending exception
2261   Label no_pending_exception;
2262   __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
2263   __ cbz(rscratch1, no_pending_exception);
2264   __ stop("must not have pending exception here");
2265   __ bind(no_pending_exception);
2266 #endif
2267 
2268   __ bind(cont);
2269 
2270   // Call C code.  Need thread and this frame, but NOT official VM entry
2271   // crud.  We cannot block on this call, no GC can happen.
2272   //
2273   // UnrollBlock* fetch_unroll_info(JavaThread* thread)
2274 
2275   // fetch_unroll_info needs to call last_java_frame().
2276 
2277   Label retaddr;
2278   __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2279 #ifdef ASSERT
2280   { Label L;
2281     __ ldr(rscratch1, Address(rthread, JavaThread::last_Java_fp_offset()));
2282     __ cbz(rscratch1, L);
2283     __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared");
2284     __ bind(L);
2285   }
2286 #endif // ASSERT
2287   __ mov(c_rarg0, rthread);
2288   __ mov(c_rarg1, rcpool);
2289   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info)));
2290   __ blr(rscratch1);
2291   __ bind(retaddr);
2292 
2293   // Need to have an oopmap that tells fetch_unroll_info where to
2294   // find any register it might need.
2295   oop_maps->add_gc_map(__ pc() - start, map);
2296 
2297   __ reset_last_Java_frame(false);
2298 
2299 #if INCLUDE_JVMCI
2300   if (EnableJVMCI) {
2301     __ bind(after_fetch_unroll_info_call);
2302   }
2303 #endif
2304 
2305   // Load UnrollBlock* into r5
2306   __ mov(r5, r0);
2307 
2308   __ ldrw(rcpool, Address(r5, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes()));
2309    Label noException;
2310   __ cmpw(rcpool, Deoptimization::Unpack_exception);   // Was exception pending?
2311   __ br(Assembler::NE, noException);
2312   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
2313   // QQQ this is useless it was NULL above
2314   __ ldr(r3, Address(rthread, JavaThread::exception_pc_offset()));
2315   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
2316   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
2317 
2318   __ verify_oop(r0);
2319 
2320   // Overwrite the result registers with the exception results.
2321   __ str(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2322   // I think this is useless
2323   // __ str(r3, Address(sp, RegisterSaver::r3_offset_in_bytes()));
2324 
2325   __ bind(noException);
2326 
2327   // Only register save data is on the stack.
2328   // Now restore the result registers.  Everything else is either dead
2329   // or captured in the vframeArray.
2330 
2331   // Restore fp result register
2332   __ ldrd(v0, Address(sp, reg_save.v0_offset_in_bytes()));
2333   // Restore integer result register
2334   __ ldr(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2335 
2336   // Pop all of the register save area off the stack
2337   __ add(sp, sp, frame_size_in_words * wordSize);
2338 
2339   // All of the register save area has been popped of the stack. Only the
2340   // return address remains.
2341 
2342   // Pop all the frames we must move/replace.
2343   //
2344   // Frame picture (youngest to oldest)
2345   // 1: self-frame (no frame link)
2346   // 2: deopting frame  (no frame link)
2347   // 3: caller of deopting frame (could be compiled/interpreted).
2348   //
2349   // Note: by leaving the return address of self-frame on the stack
2350   // and using the size of frame 2 to adjust the stack
2351   // when we are done the return to frame 3 will still be on the stack.
2352 
2353   // Pop deoptimized frame
2354   __ ldrw(r2, Address(r5, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset_in_bytes()));
2355   __ sub(r2, r2, 2 * wordSize);
2356   __ add(sp, sp, r2);
2357   __ ldp(rfp, lr, __ post(sp, 2 * wordSize));
2358   __ authenticate_return_address();
2359   // LR should now be the return address to the caller (3)
2360 
2361 #ifdef ASSERT
2362   // Compilers generate code that bang the stack by as much as the
2363   // interpreter would need. So this stack banging should never
2364   // trigger a fault. Verify that it does not on non product builds.
2365   __ ldrw(r19, Address(r5, Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes()));
2366   __ bang_stack_size(r19, r2);
2367 #endif
2368   // Load address of array of frame pcs into r2
2369   __ ldr(r2, Address(r5, Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
2370 
2371   // Trash the old pc
2372   // __ addptr(sp, wordSize);  FIXME ????
2373 
2374   // Load address of array of frame sizes into r4
2375   __ ldr(r4, Address(r5, Deoptimization::UnrollBlock::frame_sizes_offset_in_bytes()));
2376 
2377   // Load counter into r3
2378   __ ldrw(r3, Address(r5, Deoptimization::UnrollBlock::number_of_frames_offset_in_bytes()));
2379 
2380   // Now adjust the caller's stack to make up for the extra locals
2381   // but record the original sp so that we can save it in the skeletal interpreter
2382   // frame and the stack walking of interpreter_sender will get the unextended sp
2383   // value and not the "real" sp value.
2384 
2385   const Register sender_sp = r6;
2386 
2387   __ mov(sender_sp, sp);
2388   __ ldrw(r19, Address(r5,
2389                        Deoptimization::UnrollBlock::
2390                        caller_adjustment_offset_in_bytes()));
2391   __ sub(sp, sp, r19);
2392 
2393   // Push interpreter frames in a loop
2394   __ mov(rscratch1, (uint64_t)0xDEADDEAD);        // Make a recognizable pattern
2395   __ mov(rscratch2, rscratch1);
2396   Label loop;
2397   __ bind(loop);
2398   __ ldr(r19, Address(__ post(r4, wordSize)));          // Load frame size
2399   __ sub(r19, r19, 2*wordSize);           // We'll push pc and fp by hand
2400   __ ldr(lr, Address(__ post(r2, wordSize)));  // Load pc
2401   __ enter();                           // Save old & set new fp
2402   __ sub(sp, sp, r19);                  // Prolog
2403   // This value is corrected by layout_activation_impl
2404   __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
2405   __ str(sender_sp, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize)); // Make it walkable
2406   __ mov(sender_sp, sp);               // Pass sender_sp to next frame
2407   __ sub(r3, r3, 1);                   // Decrement counter
2408   __ cbnz(r3, loop);
2409 
2410     // Re-push self-frame
2411   __ ldr(lr, Address(r2));
2412   __ enter();
2413 
2414   // Allocate a full sized register save area.  We subtract 2 because
2415   // enter() just pushed 2 words
2416   __ sub(sp, sp, (frame_size_in_words - 2) * wordSize);
2417 
2418   // Restore frame locals after moving the frame
2419   __ strd(v0, Address(sp, reg_save.v0_offset_in_bytes()));
2420   __ str(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2421 
2422   // Call C code.  Need thread but NOT official VM entry
2423   // crud.  We cannot block on this call, no GC can happen.  Call should
2424   // restore return values to their stack-slots with the new SP.
2425   //
2426   // void Deoptimization::unpack_frames(JavaThread* thread, int exec_mode)
2427 
2428   // Use rfp because the frames look interpreted now
2429   // Don't need the precise return PC here, just precise enough to point into this code blob.
2430   address the_pc = __ pc();
2431   __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
2432 
2433   __ mov(c_rarg0, rthread);
2434   __ movw(c_rarg1, rcpool); // second arg: exec_mode
2435   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
2436   __ blr(rscratch1);
2437 
2438   // Set an oopmap for the call site
2439   // Use the same PC we used for the last java frame
2440   oop_maps->add_gc_map(the_pc - start,
2441                        new OopMap( frame_size_in_words, 0 ));
2442 
2443   // Clear fp AND pc
2444   __ reset_last_Java_frame(true);
2445 
2446   // Collect return values
2447   __ ldrd(v0, Address(sp, reg_save.v0_offset_in_bytes()));
2448   __ ldr(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2449   // I think this is useless (throwing pc?)
2450   // __ ldr(r3, Address(sp, RegisterSaver::r3_offset_in_bytes()));
2451 
2452   // Pop self-frame.
2453   __ leave();                           // Epilog
2454 
2455   // Jump to interpreter
2456   __ ret(lr);
2457 
2458   // Make sure all code is generated
2459   masm->flush();
2460 
2461   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
2462   _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
2463 #if INCLUDE_JVMCI
2464   if (EnableJVMCI) {
2465     _deopt_blob->set_uncommon_trap_offset(uncommon_trap_offset);
2466     _deopt_blob->set_implicit_exception_uncommon_trap_offset(implicit_exception_uncommon_trap_offset);
2467   }
2468 #endif
2469 }
2470 
2471 // Number of stack slots between incoming argument block and the start of
2472 // a new frame.  The PROLOG must add this many slots to the stack.  The
2473 // EPILOG must remove this many slots. aarch64 needs two slots for
2474 // return address and fp.
2475 // TODO think this is correct but check
2476 uint SharedRuntime::in_preserve_stack_slots() {
2477   return 4;
2478 }
2479 
2480 uint SharedRuntime::out_preserve_stack_slots() {
2481   return 0;
2482 }
2483 
2484 #ifdef COMPILER2
2485 //------------------------------generate_uncommon_trap_blob--------------------
2486 void SharedRuntime::generate_uncommon_trap_blob() {
2487   // Allocate space for the code
2488   ResourceMark rm;
2489   // Setup code generation tools
2490   CodeBuffer buffer("uncommon_trap_blob", 2048, 1024);
2491   MacroAssembler* masm = new MacroAssembler(&buffer);
2492 
2493   assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
2494 
2495   address start = __ pc();
2496 
2497   // Push self-frame.  We get here with a return address in LR
2498   // and sp should be 16 byte aligned
2499   // push rfp and retaddr by hand
2500   __ protect_return_address();
2501   __ stp(rfp, lr, Address(__ pre(sp, -2 * wordSize)));
2502   // we don't expect an arg reg save area
2503 #ifndef PRODUCT
2504   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
2505 #endif
2506   // compiler left unloaded_class_index in j_rarg0 move to where the
2507   // runtime expects it.
2508   if (c_rarg1 != j_rarg0) {
2509     __ movw(c_rarg1, j_rarg0);
2510   }
2511 
2512   // we need to set the past SP to the stack pointer of the stub frame
2513   // and the pc to the address where this runtime call will return
2514   // although actually any pc in this code blob will do).
2515   Label retaddr;
2516   __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2517 
2518   // Call C code.  Need thread but NOT official VM entry
2519   // crud.  We cannot block on this call, no GC can happen.  Call should
2520   // capture callee-saved registers as well as return values.
2521   // Thread is in rdi already.
2522   //
2523   // UnrollBlock* uncommon_trap(JavaThread* thread, jint unloaded_class_index);
2524   //
2525   // n.b. 2 gp args, 0 fp args, integral return type
2526 
2527   __ mov(c_rarg0, rthread);
2528   __ movw(c_rarg2, (unsigned)Deoptimization::Unpack_uncommon_trap);
2529   __ lea(rscratch1,
2530          RuntimeAddress(CAST_FROM_FN_PTR(address,
2531                                          Deoptimization::uncommon_trap)));
2532   __ blr(rscratch1);
2533   __ bind(retaddr);
2534 
2535   // Set an oopmap for the call site
2536   OopMapSet* oop_maps = new OopMapSet();
2537   OopMap* map = new OopMap(SimpleRuntimeFrame::framesize, 0);
2538 
2539   // location of rfp is known implicitly by the frame sender code
2540 
2541   oop_maps->add_gc_map(__ pc() - start, map);
2542 
2543   __ reset_last_Java_frame(false);
2544 
2545   // move UnrollBlock* into r4
2546   __ mov(r4, r0);
2547 
2548 #ifdef ASSERT
2549   { Label L;
2550     __ ldrw(rscratch1, Address(r4, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes()));
2551     __ cmpw(rscratch1, (unsigned)Deoptimization::Unpack_uncommon_trap);
2552     __ br(Assembler::EQ, L);
2553     __ stop("SharedRuntime::generate_uncommon_trap_blob: expected Unpack_uncommon_trap");
2554     __ bind(L);
2555   }
2556 #endif
2557 
2558   // Pop all the frames we must move/replace.
2559   //
2560   // Frame picture (youngest to oldest)
2561   // 1: self-frame (no frame link)
2562   // 2: deopting frame  (no frame link)
2563   // 3: caller of deopting frame (could be compiled/interpreted).
2564 
2565   // Pop self-frame.  We have no frame, and must rely only on r0 and sp.
2566   __ add(sp, sp, (SimpleRuntimeFrame::framesize) << LogBytesPerInt); // Epilog!
2567 
2568   // Pop deoptimized frame (int)
2569   __ ldrw(r2, Address(r4,
2570                       Deoptimization::UnrollBlock::
2571                       size_of_deoptimized_frame_offset_in_bytes()));
2572   __ sub(r2, r2, 2 * wordSize);
2573   __ add(sp, sp, r2);
2574   __ ldp(rfp, lr, __ post(sp, 2 * wordSize));
2575   __ authenticate_return_address();
2576   // LR should now be the return address to the caller (3) frame
2577 
2578 #ifdef ASSERT
2579   // Compilers generate code that bang the stack by as much as the
2580   // interpreter would need. So this stack banging should never
2581   // trigger a fault. Verify that it does not on non product builds.
2582   __ ldrw(r1, Address(r4,
2583                       Deoptimization::UnrollBlock::
2584                       total_frame_sizes_offset_in_bytes()));
2585   __ bang_stack_size(r1, r2);
2586 #endif
2587 
2588   // Load address of array of frame pcs into r2 (address*)
2589   __ ldr(r2, Address(r4,
2590                      Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
2591 
2592   // Load address of array of frame sizes into r5 (intptr_t*)
2593   __ ldr(r5, Address(r4,
2594                      Deoptimization::UnrollBlock::
2595                      frame_sizes_offset_in_bytes()));
2596 
2597   // Counter
2598   __ ldrw(r3, Address(r4,
2599                       Deoptimization::UnrollBlock::
2600                       number_of_frames_offset_in_bytes())); // (int)
2601 
2602   // Now adjust the caller's stack to make up for the extra locals but
2603   // record the original sp so that we can save it in the skeletal
2604   // interpreter frame and the stack walking of interpreter_sender
2605   // will get the unextended sp value and not the "real" sp value.
2606 
2607   const Register sender_sp = r8;
2608 
2609   __ mov(sender_sp, sp);
2610   __ ldrw(r1, Address(r4,
2611                       Deoptimization::UnrollBlock::
2612                       caller_adjustment_offset_in_bytes())); // (int)
2613   __ sub(sp, sp, r1);
2614 
2615   // Push interpreter frames in a loop
2616   Label loop;
2617   __ bind(loop);
2618   __ ldr(r1, Address(r5, 0));       // Load frame size
2619   __ sub(r1, r1, 2 * wordSize);     // We'll push pc and rfp by hand
2620   __ ldr(lr, Address(r2, 0));       // Save return address
2621   __ enter();                       // and old rfp & set new rfp
2622   __ sub(sp, sp, r1);               // Prolog
2623   __ str(sender_sp, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize)); // Make it walkable
2624   // This value is corrected by layout_activation_impl
2625   __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
2626   __ mov(sender_sp, sp);          // Pass sender_sp to next frame
2627   __ add(r5, r5, wordSize);       // Bump array pointer (sizes)
2628   __ add(r2, r2, wordSize);       // Bump array pointer (pcs)
2629   __ subsw(r3, r3, 1);            // Decrement counter
2630   __ br(Assembler::GT, loop);
2631   __ ldr(lr, Address(r2, 0));     // save final return address
2632   // Re-push self-frame
2633   __ enter();                     // & old rfp & set new rfp
2634 
2635   // Use rfp because the frames look interpreted now
2636   // Save "the_pc" since it cannot easily be retrieved using the last_java_SP after we aligned SP.
2637   // Don't need the precise return PC here, just precise enough to point into this code blob.
2638   address the_pc = __ pc();
2639   __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
2640 
2641   // Call C code.  Need thread but NOT official VM entry
2642   // crud.  We cannot block on this call, no GC can happen.  Call should
2643   // restore return values to their stack-slots with the new SP.
2644   // Thread is in rdi already.
2645   //
2646   // BasicType unpack_frames(JavaThread* thread, int exec_mode);
2647   //
2648   // n.b. 2 gp args, 0 fp args, integral return type
2649 
2650   // sp should already be aligned
2651   __ mov(c_rarg0, rthread);
2652   __ movw(c_rarg1, (unsigned)Deoptimization::Unpack_uncommon_trap);
2653   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
2654   __ blr(rscratch1);
2655 
2656   // Set an oopmap for the call site
2657   // Use the same PC we used for the last java frame
2658   oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0));
2659 
2660   // Clear fp AND pc
2661   __ reset_last_Java_frame(true);
2662 
2663   // Pop self-frame.
2664   __ leave();                 // Epilog
2665 
2666   // Jump to interpreter
2667   __ ret(lr);
2668 
2669   // Make sure all code is generated
2670   masm->flush();
2671 
2672   _uncommon_trap_blob =  UncommonTrapBlob::create(&buffer, oop_maps,
2673                                                  SimpleRuntimeFrame::framesize >> 1);
2674 }
2675 #endif // COMPILER2
2676 
2677 
2678 //------------------------------generate_handler_blob------
2679 //
2680 // Generate a special Compile2Runtime blob that saves all registers,
2681 // and setup oopmap.
2682 //
2683 SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_type) {
2684   ResourceMark rm;
2685   OopMapSet *oop_maps = new OopMapSet();
2686   OopMap* map;
2687 
2688   // Allocate space for the code.  Setup code generation tools.
2689   CodeBuffer buffer("handler_blob", 2048, 1024);
2690   MacroAssembler* masm = new MacroAssembler(&buffer);
2691 
2692   address start   = __ pc();
2693   address call_pc = NULL;
2694   int frame_size_in_words;
2695   bool cause_return = (poll_type == POLL_AT_RETURN);
2696   RegisterSaver reg_save(poll_type == POLL_AT_VECTOR_LOOP /* save_vectors */);
2697 
2698   // When the signal occurred, the LR was either signed and stored on the stack (in which
2699   // case it will be restored from the stack before being used) or unsigned and not stored
2700   // on the stack. Stipping ensures we get the right value.
2701   __ strip_return_address();
2702 
2703   // Save Integer and Float registers.
2704   map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2705 
2706   // The following is basically a call_VM.  However, we need the precise
2707   // address of the call in order to generate an oopmap. Hence, we do all the
2708   // work ourselves.
2709 
2710   Label retaddr;
2711   __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2712 
2713   // The return address must always be correct so that frame constructor never
2714   // sees an invalid pc.
2715 
2716   if (!cause_return) {
2717     // overwrite the return address pushed by save_live_registers
2718     // Additionally, r20 is a callee-saved register so we can look at
2719     // it later to determine if someone changed the return address for
2720     // us!
2721     __ ldr(r20, Address(rthread, JavaThread::saved_exception_pc_offset()));
2722     __ protect_return_address(r20, rscratch1);
2723     __ str(r20, Address(rfp, wordSize));
2724   }
2725 
2726   // Do the call
2727   __ mov(c_rarg0, rthread);
2728   __ lea(rscratch1, RuntimeAddress(call_ptr));
2729   __ blr(rscratch1);
2730   __ bind(retaddr);
2731 
2732   // Set an oopmap for the call site.  This oopmap will map all
2733   // oop-registers and debug-info registers as callee-saved.  This
2734   // will allow deoptimization at this safepoint to find all possible
2735   // debug-info recordings, as well as let GC find all oops.
2736 
2737   oop_maps->add_gc_map( __ pc() - start, map);
2738 
2739   Label noException;
2740 
2741   __ reset_last_Java_frame(false);
2742 
2743   __ membar(Assembler::LoadLoad | Assembler::LoadStore);
2744 
2745   __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
2746   __ cbz(rscratch1, noException);
2747 
2748   // Exception pending
2749 
2750   reg_save.restore_live_registers(masm);
2751 
2752   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2753 
2754   // No exception case
2755   __ bind(noException);
2756 
2757   Label no_adjust, bail;
2758   if (!cause_return) {
2759     // If our stashed return pc was modified by the runtime we avoid touching it
2760     __ ldr(rscratch1, Address(rfp, wordSize));
2761     __ cmp(r20, rscratch1);
2762     __ br(Assembler::NE, no_adjust);
2763     __ authenticate_return_address(r20, rscratch1);
2764 
2765 #ifdef ASSERT
2766     // Verify the correct encoding of the poll we're about to skip.
2767     // See NativeInstruction::is_ldrw_to_zr()
2768     __ ldrw(rscratch1, Address(r20));
2769     __ ubfx(rscratch2, rscratch1, 22, 10);
2770     __ cmpw(rscratch2, 0b1011100101);
2771     __ br(Assembler::NE, bail);
2772     __ ubfx(rscratch2, rscratch1, 0, 5);
2773     __ cmpw(rscratch2, 0b11111);
2774     __ br(Assembler::NE, bail);
2775 #endif
2776     // Adjust return pc forward to step over the safepoint poll instruction
2777     __ add(r20, r20, NativeInstruction::instruction_size);
2778     __ protect_return_address(r20, rscratch1);
2779     __ str(r20, Address(rfp, wordSize));
2780   }
2781 
2782   __ bind(no_adjust);
2783   // Normal exit, restore registers and exit.
2784   reg_save.restore_live_registers(masm);
2785 
2786   __ ret(lr);
2787 
2788 #ifdef ASSERT
2789   __ bind(bail);
2790   __ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected");
2791 #endif
2792 
2793   // Make sure all code is generated
2794   masm->flush();
2795 
2796   // Fill-out other meta info
2797   return SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
2798 }
2799 
2800 //
2801 // generate_resolve_blob - call resolution (static/virtual/opt-virtual/ic-miss
2802 //
2803 // Generate a stub that calls into vm to find out the proper destination
2804 // of a java call. All the argument registers are live at this point
2805 // but since this is generic code we don't know what they are and the caller
2806 // must do any gc of the args.
2807 //
2808 RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const char* name) {
2809   assert (StubRoutines::forward_exception_entry() != NULL, "must be generated before");
2810 
2811   // allocate space for the code
2812   ResourceMark rm;
2813 
2814   CodeBuffer buffer(name, 1000, 512);
2815   MacroAssembler* masm                = new MacroAssembler(&buffer);
2816 
2817   int frame_size_in_words;
2818   RegisterSaver reg_save(false /* save_vectors */);
2819 
2820   OopMapSet *oop_maps = new OopMapSet();
2821   OopMap* map = NULL;
2822 
2823   int start = __ offset();
2824 
2825   map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2826 
2827   int frame_complete = __ offset();
2828 
2829   {
2830     Label retaddr;
2831     __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2832 
2833     __ mov(c_rarg0, rthread);
2834     __ lea(rscratch1, RuntimeAddress(destination));
2835 
2836     __ blr(rscratch1);
2837     __ bind(retaddr);
2838   }
2839 
2840   // Set an oopmap for the call site.
2841   // We need this not only for callee-saved registers, but also for volatile
2842   // registers that the compiler might be keeping live across a safepoint.
2843 
2844   oop_maps->add_gc_map( __ offset() - start, map);
2845 
2846   // r0 contains the address we are going to jump to assuming no exception got installed
2847 
2848   // clear last_Java_sp
2849   __ reset_last_Java_frame(false);
2850   // check for pending exceptions
2851   Label pending;
2852   __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
2853   __ cbnz(rscratch1, pending);
2854 
2855   // get the returned Method*
2856   __ get_vm_result_2(rmethod, rthread);
2857   __ str(rmethod, Address(sp, reg_save.reg_offset_in_bytes(rmethod)));
2858 
2859   // r0 is where we want to jump, overwrite rscratch1 which is saved and scratch
2860   __ str(r0, Address(sp, reg_save.rscratch1_offset_in_bytes()));
2861   reg_save.restore_live_registers(masm);
2862 
2863   // We are back to the original state on entry and ready to go.
2864 
2865   __ br(rscratch1);
2866 
2867   // Pending exception after the safepoint
2868 
2869   __ bind(pending);
2870 
2871   reg_save.restore_live_registers(masm);
2872 
2873   // exception pending => remove activation and forward to exception handler
2874 
2875   __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
2876 
2877   __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
2878   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2879 
2880   // -------------
2881   // make sure all code is generated
2882   masm->flush();
2883 
2884   // return the  blob
2885   // frame_size_words or bytes??
2886   return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true);
2887 }
2888 
2889 #ifdef COMPILER2
2890 // This is here instead of runtime_aarch64_64.cpp because it uses SimpleRuntimeFrame
2891 //
2892 //------------------------------generate_exception_blob---------------------------
2893 // creates exception blob at the end
2894 // Using exception blob, this code is jumped from a compiled method.
2895 // (see emit_exception_handler in x86_64.ad file)
2896 //
2897 // Given an exception pc at a call we call into the runtime for the
2898 // handler in this method. This handler might merely restore state
2899 // (i.e. callee save registers) unwind the frame and jump to the
2900 // exception handler for the nmethod if there is no Java level handler
2901 // for the nmethod.
2902 //
2903 // This code is entered with a jmp.
2904 //
2905 // Arguments:
2906 //   r0: exception oop
2907 //   r3: exception pc
2908 //
2909 // Results:
2910 //   r0: exception oop
2911 //   r3: exception pc in caller or ???
2912 //   destination: exception handler of caller
2913 //
2914 // Note: the exception pc MUST be at a call (precise debug information)
2915 //       Registers r0, r3, r2, r4, r5, r8-r11 are not callee saved.
2916 //
2917 
2918 void OptoRuntime::generate_exception_blob() {
2919   assert(!OptoRuntime::is_callee_saved_register(R3_num), "");
2920   assert(!OptoRuntime::is_callee_saved_register(R0_num), "");
2921   assert(!OptoRuntime::is_callee_saved_register(R2_num), "");
2922 
2923   assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
2924 
2925   // Allocate space for the code
2926   ResourceMark rm;
2927   // Setup code generation tools
2928   CodeBuffer buffer("exception_blob", 2048, 1024);
2929   MacroAssembler* masm = new MacroAssembler(&buffer);
2930 
2931   // TODO check various assumptions made here
2932   //
2933   // make sure we do so before running this
2934 
2935   address start = __ pc();
2936 
2937   // push rfp and retaddr by hand
2938   // Exception pc is 'return address' for stack walker
2939   __ protect_return_address();
2940   __ stp(rfp, lr, Address(__ pre(sp, -2 * wordSize)));
2941   // there are no callee save registers and we don't expect an
2942   // arg reg save area
2943 #ifndef PRODUCT
2944   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
2945 #endif
2946   // Store exception in Thread object. We cannot pass any arguments to the
2947   // handle_exception call, since we do not want to make any assumption
2948   // about the size of the frame where the exception happened in.
2949   __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
2950   __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
2951 
2952   // This call does all the hard work.  It checks if an exception handler
2953   // exists in the method.
2954   // If so, it returns the handler address.
2955   // If not, it prepares for stack-unwinding, restoring the callee-save
2956   // registers of the frame being removed.
2957   //
2958   // address OptoRuntime::handle_exception_C(JavaThread* thread)
2959   //
2960   // n.b. 1 gp arg, 0 fp args, integral return type
2961 
2962   // the stack should always be aligned
2963   address the_pc = __ pc();
2964   __ set_last_Java_frame(sp, noreg, the_pc, rscratch1);
2965   __ mov(c_rarg0, rthread);
2966   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, OptoRuntime::handle_exception_C)));
2967   __ blr(rscratch1);
2968   // handle_exception_C is a special VM call which does not require an explicit
2969   // instruction sync afterwards.
2970 
2971   // May jump to SVE compiled code
2972   __ reinitialize_ptrue();
2973 
2974   // Set an oopmap for the call site.  This oopmap will only be used if we
2975   // are unwinding the stack.  Hence, all locations will be dead.
2976   // Callee-saved registers will be the same as the frame above (i.e.,
2977   // handle_exception_stub), since they were restored when we got the
2978   // exception.
2979 
2980   OopMapSet* oop_maps = new OopMapSet();
2981 
2982   oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0));
2983 
2984   __ reset_last_Java_frame(false);
2985 
2986   // Restore callee-saved registers
2987 
2988   // rfp is an implicitly saved callee saved register (i.e. the calling
2989   // convention will save restore it in prolog/epilog) Other than that
2990   // there are no callee save registers now that adapter frames are gone.
2991   // and we dont' expect an arg reg save area
2992   __ ldp(rfp, r3, Address(__ post(sp, 2 * wordSize)));
2993   __ authenticate_return_address(r3);
2994 
2995   // r0: exception handler
2996 
2997   // We have a handler in r0 (could be deopt blob).
2998   __ mov(r8, r0);
2999 
3000   // Get the exception oop
3001   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
3002   // Get the exception pc in case we are deoptimized
3003   __ ldr(r4, Address(rthread, JavaThread::exception_pc_offset()));
3004 #ifdef ASSERT
3005   __ str(zr, Address(rthread, JavaThread::exception_handler_pc_offset()));
3006   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
3007 #endif
3008   // Clear the exception oop so GC no longer processes it as a root.
3009   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
3010 
3011   // r0: exception oop
3012   // r8:  exception handler
3013   // r4: exception pc
3014   // Jump to handler
3015 
3016   __ br(r8);
3017 
3018   // Make sure all code is generated
3019   masm->flush();
3020 
3021   // Set exception blob
3022   _exception_blob =  ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
3023 }
3024 
3025 #endif // COMPILER2