1 /*
   2  * Copyright (c) 2003, 2021, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2021, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2021, Azul Systems, Inc. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"
  30 #include "classfile/symbolTable.hpp"
  31 #include "code/codeCache.hpp"
  32 #include "code/debugInfoRec.hpp"
  33 #include "code/icBuffer.hpp"
  34 #include "code/vtableStubs.hpp"
  35 #include "compiler/oopMap.hpp"
  36 #include "gc/shared/barrierSetAssembler.hpp"
  37 #include "interpreter/interpreter.hpp"
  38 #include "interpreter/interp_masm.hpp"
  39 #include "logging/log.hpp"
  40 #include "memory/resourceArea.hpp"
  41 #include "nativeInst_aarch64.hpp"
  42 #include "oops/compiledICHolder.hpp"
  43 #include "oops/klass.inline.hpp"
  44 #include "prims/methodHandles.hpp"
  45 #include "runtime/jniHandles.hpp"
  46 #include "runtime/safepointMechanism.hpp"
  47 #include "runtime/sharedRuntime.hpp"
  48 #include "runtime/signature.hpp"
  49 #include "runtime/stubRoutines.hpp"
  50 #include "runtime/vframeArray.hpp"
  51 #include "utilities/align.hpp"
  52 #include "utilities/formatBuffer.hpp"
  53 #include "vmreg_aarch64.inline.hpp"
  54 #ifdef COMPILER1
  55 #include "c1/c1_Runtime1.hpp"
  56 #endif
  57 #ifdef COMPILER2
  58 #include "adfiles/ad_aarch64.hpp"
  59 #include "opto/runtime.hpp"
  60 #endif
  61 #if INCLUDE_JVMCI
  62 #include "jvmci/jvmciJavaClasses.hpp"
  63 #endif
  64 
  65 #define __ masm->
  66 
  67 const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;
  68 
  69 class SimpleRuntimeFrame {
  70 
  71   public:
  72 
  73   // Most of the runtime stubs have this simple frame layout.
  74   // This class exists to make the layout shared in one place.
  75   // Offsets are for compiler stack slots, which are jints.
  76   enum layout {
  77     // The frame sender code expects that rbp will be in the "natural" place and
  78     // will override any oopMap setting for it. We must therefore force the layout
  79     // so that it agrees with the frame sender code.
  80     // we don't expect any arg reg save area so aarch64 asserts that
  81     // frame::arg_reg_save_area_bytes == 0
  82     rbp_off = 0,
  83     rbp_off2,
  84     return_off, return_off2,
  85     framesize
  86   };
  87 };
  88 
  89 // FIXME -- this is used by C1
  90 class RegisterSaver {
  91   const bool _save_vectors;
  92  public:
  93   RegisterSaver(bool save_vectors) : _save_vectors(save_vectors) {}
  94 
  95   OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words);
  96   void restore_live_registers(MacroAssembler* masm);
  97 
  98   // Offsets into the register save area
  99   // Used by deoptimization when it is managing result register
 100   // values on its own
 101 
 102   int reg_offset_in_bytes(Register r);
 103   int r0_offset_in_bytes()    { return reg_offset_in_bytes(r0); }
 104   int rscratch1_offset_in_bytes()    { return reg_offset_in_bytes(rscratch1); }
 105   int v0_offset_in_bytes(void)   { return 0; }
 106 
 107   // Capture info about frame layout
 108   // Note this is only correct when not saving full vectors.
 109   enum layout {
 110                 fpu_state_off = 0,
 111                 fpu_state_end = fpu_state_off + FPUStateSizeInWords - 1,
 112                 // The frame sender code expects that rfp will be in
 113                 // the "natural" place and will override any oopMap
 114                 // setting for it. We must therefore force the layout
 115                 // so that it agrees with the frame sender code.
 116                 r0_off = fpu_state_off + FPUStateSizeInWords,
 117                 rfp_off = r0_off + (RegisterImpl::number_of_registers - 2) * RegisterImpl::max_slots_per_register,
 118                 return_off = rfp_off + RegisterImpl::max_slots_per_register,      // slot for return address
 119                 reg_save_size = return_off + RegisterImpl::max_slots_per_register};
 120 
 121 };
 122 
 123 int RegisterSaver::reg_offset_in_bytes(Register r) {
 124   // The integer registers are located above the floating point
 125   // registers in the stack frame pushed by save_live_registers() so the
 126   // offset depends on whether we are saving full vectors, and whether
 127   // those vectors are NEON or SVE.
 128 
 129   int slots_per_vect = FloatRegisterImpl::save_slots_per_register;
 130 
 131 #if COMPILER2_OR_JVMCI
 132   if (_save_vectors) {
 133     slots_per_vect = FloatRegisterImpl::slots_per_neon_register;
 134 
 135 #ifdef COMPILER2
 136     if (Matcher::supports_scalable_vector()) {
 137       slots_per_vect = Matcher::scalable_vector_reg_size(T_FLOAT);
 138     }
 139 #endif
 140   }
 141 #endif
 142 
 143   int r0_offset = (slots_per_vect * FloatRegisterImpl::number_of_registers) * BytesPerInt;
 144   return r0_offset + r->encoding() * wordSize;
 145 }
 146 
 147 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words) {
 148   bool use_sve = false;
 149   int sve_vector_size_in_bytes = 0;
 150   int sve_vector_size_in_slots = 0;
 151 
 152 #ifdef COMPILER2
 153   use_sve = Matcher::supports_scalable_vector();
 154   sve_vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
 155   sve_vector_size_in_slots = Matcher::scalable_vector_reg_size(T_FLOAT);
 156 #endif
 157 
 158 #if COMPILER2_OR_JVMCI
 159   if (_save_vectors) {
 160     int vect_words = 0;
 161     int extra_save_slots_per_register = 0;
 162     // Save upper half of vector registers
 163     if (use_sve) {
 164       extra_save_slots_per_register = sve_vector_size_in_slots - FloatRegisterImpl::save_slots_per_register;
 165     } else {
 166       extra_save_slots_per_register = FloatRegisterImpl::extra_save_slots_per_neon_register;
 167     }
 168     vect_words = FloatRegisterImpl::number_of_registers * extra_save_slots_per_register /
 169                  VMRegImpl::slots_per_word;
 170     additional_frame_words += vect_words;
 171   }
 172 #else
 173   assert(!_save_vectors, "vectors are generated only by C2 and JVMCI");
 174 #endif
 175 
 176   int frame_size_in_bytes = align_up(additional_frame_words * wordSize +
 177                                      reg_save_size * BytesPerInt, 16);
 178   // OopMap frame size is in compiler stack slots (jint's) not bytes or words
 179   int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
 180   // The caller will allocate additional_frame_words
 181   int additional_frame_slots = additional_frame_words * wordSize / BytesPerInt;
 182   // CodeBlob frame size is in words.
 183   int frame_size_in_words = frame_size_in_bytes / wordSize;
 184   *total_frame_words = frame_size_in_words;
 185 
 186   // Save Integer and Float registers.
 187   __ enter();
 188   __ push_CPU_state(_save_vectors, use_sve, sve_vector_size_in_bytes);
 189 
 190   // Set an oopmap for the call site.  This oopmap will map all
 191   // oop-registers and debug-info registers as callee-saved.  This
 192   // will allow deoptimization at this safepoint to find all possible
 193   // debug-info recordings, as well as let GC find all oops.
 194 
 195   OopMapSet *oop_maps = new OopMapSet();
 196   OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
 197 
 198   for (int i = 0; i < RegisterImpl::number_of_registers; i++) {
 199     Register r = as_Register(i);
 200     if (r <= rfp && r != rscratch1 && r != rscratch2) {
 201       // SP offsets are in 4-byte words.
 202       // Register slots are 8 bytes wide, 32 floating-point registers.
 203       int sp_offset = RegisterImpl::max_slots_per_register * i +
 204                       FloatRegisterImpl::save_slots_per_register * FloatRegisterImpl::number_of_registers;
 205       oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset + additional_frame_slots),
 206                                 r->as_VMReg());
 207     }
 208   }
 209 
 210   for (int i = 0; i < FloatRegisterImpl::number_of_registers; i++) {
 211     FloatRegister r = as_FloatRegister(i);
 212     int sp_offset = 0;
 213     if (_save_vectors) {
 214       sp_offset = use_sve ? (sve_vector_size_in_slots * i) :
 215                             (FloatRegisterImpl::slots_per_neon_register * i);
 216     } else {
 217       sp_offset = FloatRegisterImpl::save_slots_per_register * i;
 218     }
 219     oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
 220                               r->as_VMReg());
 221   }
 222 
 223   return oop_map;
 224 }
 225 
 226 void RegisterSaver::restore_live_registers(MacroAssembler* masm) {
 227 #ifdef COMPILER2
 228   __ pop_CPU_state(_save_vectors, Matcher::supports_scalable_vector(),
 229                    Matcher::scalable_vector_reg_size(T_BYTE));
 230 #else
 231 #if !INCLUDE_JVMCI
 232   assert(!_save_vectors, "vectors are generated only by C2 and JVMCI");
 233 #endif
 234   __ pop_CPU_state(_save_vectors);
 235 #endif
 236   __ leave();
 237 
 238 }
 239 
 240 // Is vector's size (in bytes) bigger than a size saved by default?
 241 // 8 bytes vector registers are saved by default on AArch64.
 242 bool SharedRuntime::is_wide_vector(int size) {
 243   return size > 8;
 244 }
 245 
 246 // The java_calling_convention describes stack locations as ideal slots on
 247 // a frame with no abi restrictions. Since we must observe abi restrictions
 248 // (like the placement of the register window) the slots must be biased by
 249 // the following value.
 250 static int reg2offset_in(VMReg r) {
 251   // Account for saved rfp and lr
 252   // This should really be in_preserve_stack_slots
 253   return (r->reg2stack() + 4) * VMRegImpl::stack_slot_size;
 254 }
 255 
 256 static int reg2offset_out(VMReg r) {
 257   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 258 }
 259 
 260 // ---------------------------------------------------------------------------
 261 // Read the array of BasicTypes from a signature, and compute where the
 262 // arguments should go.  Values in the VMRegPair regs array refer to 4-byte
 263 // quantities.  Values less than VMRegImpl::stack0 are registers, those above
 264 // refer to 4-byte stack slots.  All stack slots are based off of the stack pointer
 265 // as framesizes are fixed.
 266 // VMRegImpl::stack0 refers to the first slot 0(sp).
 267 // and VMRegImpl::stack0+1 refers to the memory word 4-byes higher.  Register
 268 // up to RegisterImpl::number_of_registers) are the 64-bit
 269 // integer registers.
 270 
 271 // Note: the INPUTS in sig_bt are in units of Java argument words,
 272 // which are 64-bit.  The OUTPUTS are in 32-bit units.
 273 
 274 // The Java calling convention is a "shifted" version of the C ABI.
 275 // By skipping the first C ABI register we can call non-static jni
 276 // methods with small numbers of arguments without having to shuffle
 277 // the arguments at all. Since we control the java ABI we ought to at
 278 // least get some advantage out of it.
 279 
 280 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
 281                                            VMRegPair *regs,
 282                                            int total_args_passed) {
 283 
 284   // Create the mapping between argument positions and
 285   // registers.
 286   static const Register INT_ArgReg[Argument::n_int_register_parameters_j] = {
 287     j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5, j_rarg6, j_rarg7
 288   };
 289   static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_j] = {
 290     j_farg0, j_farg1, j_farg2, j_farg3,
 291     j_farg4, j_farg5, j_farg6, j_farg7
 292   };
 293 
 294 
 295   uint int_args = 0;
 296   uint fp_args = 0;
 297   uint stk_args = 0; // inc by 2 each time
 298 
 299   for (int i = 0; i < total_args_passed; i++) {
 300     switch (sig_bt[i]) {
 301     case T_BOOLEAN:
 302     case T_CHAR:
 303     case T_BYTE:
 304     case T_SHORT:
 305     case T_INT:
 306       if (int_args < Argument::n_int_register_parameters_j) {
 307         regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
 308       } else {
 309         regs[i].set1(VMRegImpl::stack2reg(stk_args));
 310         stk_args += 2;
 311       }
 312       break;
 313     case T_VOID:
 314       // halves of T_LONG or T_DOUBLE
 315       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 316       regs[i].set_bad();
 317       break;
 318     case T_LONG:
 319       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 320       // fall through
 321     case T_OBJECT:
 322     case T_ARRAY:
 323     case T_ADDRESS:
 324     case T_INLINE_TYPE:
 325       if (int_args < Argument::n_int_register_parameters_j) {
 326         regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
 327       } else {
 328         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 329         stk_args += 2;
 330       }
 331       break;
 332     case T_FLOAT:
 333       if (fp_args < Argument::n_float_register_parameters_j) {
 334         regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
 335       } else {
 336         regs[i].set1(VMRegImpl::stack2reg(stk_args));
 337         stk_args += 2;
 338       }
 339       break;
 340     case T_DOUBLE:
 341       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 342       if (fp_args < Argument::n_float_register_parameters_j) {
 343         regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 344       } else {
 345         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 346         stk_args += 2;
 347       }
 348       break;
 349     default:
 350       ShouldNotReachHere();
 351       break;
 352     }
 353   }
 354 
 355   return align_up(stk_args, 2);
 356 }
 357 
 358 
 359 const uint SharedRuntime::java_return_convention_max_int = Argument::n_int_register_parameters_j;
 360 const uint SharedRuntime::java_return_convention_max_float = Argument::n_float_register_parameters_j;
 361 
 362 int SharedRuntime::java_return_convention(const BasicType *sig_bt, VMRegPair *regs, int total_args_passed) {
 363 
 364   // Create the mapping between argument positions and registers.
 365 
 366   static const Register INT_ArgReg[java_return_convention_max_int] = {
 367     r0 /* j_rarg7 */, j_rarg6, j_rarg5, j_rarg4, j_rarg3, j_rarg2, j_rarg1, j_rarg0
 368   };
 369 
 370   static const FloatRegister FP_ArgReg[java_return_convention_max_float] = {
 371     j_farg0, j_farg1, j_farg2, j_farg3, j_farg4, j_farg5, j_farg6, j_farg7
 372   };
 373 
 374   uint int_args = 0;
 375   uint fp_args = 0;
 376 
 377   for (int i = 0; i < total_args_passed; i++) {
 378     switch (sig_bt[i]) {
 379     case T_BOOLEAN:
 380     case T_CHAR:
 381     case T_BYTE:
 382     case T_SHORT:
 383     case T_INT:
 384       if (int_args < SharedRuntime::java_return_convention_max_int) {
 385         regs[i].set1(INT_ArgReg[int_args]->as_VMReg());
 386         int_args ++;
 387       } else {
 388         return -1;
 389       }
 390       break;
 391     case T_VOID:
 392       // halves of T_LONG or T_DOUBLE
 393       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 394       regs[i].set_bad();
 395       break;
 396     case T_LONG:
 397       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 398       // fall through
 399     case T_OBJECT:
 400     case T_ARRAY:
 401     case T_ADDRESS:
 402       // Should T_METADATA be added to java_calling_convention as well ?
 403     case T_METADATA:
 404     case T_INLINE_TYPE:
 405       if (int_args < SharedRuntime::java_return_convention_max_int) {
 406         regs[i].set2(INT_ArgReg[int_args]->as_VMReg());
 407         int_args ++;
 408       } else {
 409         return -1;
 410       }
 411       break;
 412     case T_FLOAT:
 413       if (fp_args < SharedRuntime::java_return_convention_max_float) {
 414         regs[i].set1(FP_ArgReg[fp_args]->as_VMReg());
 415         fp_args ++;
 416       } else {
 417         return -1;
 418       }
 419       break;
 420     case T_DOUBLE:
 421       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 422       if (fp_args < SharedRuntime::java_return_convention_max_float) {
 423         regs[i].set2(FP_ArgReg[fp_args]->as_VMReg());
 424         fp_args ++;
 425       } else {
 426         return -1;
 427       }
 428       break;
 429     default:
 430       ShouldNotReachHere();
 431       break;
 432     }
 433   }
 434 
 435   return int_args + fp_args;
 436 }
 437 
 438 // Patch the callers callsite with entry to compiled code if it exists.
 439 static void patch_callers_callsite(MacroAssembler *masm) {
 440   Label L;
 441   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 442   __ cbz(rscratch1, L);
 443 
 444   __ enter();
 445   __ push_CPU_state();
 446 
 447   // VM needs caller's callsite
 448   // VM needs target method
 449   // This needs to be a long call since we will relocate this adapter to
 450   // the codeBuffer and it may not reach
 451 
 452 #ifndef PRODUCT
 453   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
 454 #endif
 455 
 456   __ mov(c_rarg0, rmethod);
 457   __ mov(c_rarg1, lr);
 458   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 459   __ blr(rscratch1);
 460 
 461   // Explicit isb required because fixup_callers_callsite may change the code
 462   // stream.
 463   __ safepoint_isb();
 464 
 465   __ pop_CPU_state();
 466   // restore sp
 467   __ leave();
 468   __ bind(L);
 469 }
 470 
 471 // For each inline type argument, sig includes the list of fields of
 472 // the inline type. This utility function computes the number of
 473 // arguments for the call if inline types are passed by reference (the
 474 // calling convention the interpreter expects).
 475 static int compute_total_args_passed_int(const GrowableArray<SigEntry>* sig_extended) {
 476   int total_args_passed = 0;
 477   if (InlineTypePassFieldsAsArgs) {
 478      for (int i = 0; i < sig_extended->length(); i++) {
 479        BasicType bt = sig_extended->at(i)._bt;
 480        if (bt == T_INLINE_TYPE) {
 481          // In sig_extended, an inline type argument starts with:
 482          // T_INLINE_TYPE, followed by the types of the fields of the
 483          // inline type and T_VOID to mark the end of the value
 484          // type. Inline types are flattened so, for instance, in the
 485          // case of an inline type with an int field and an inline type
 486          // field that itself has 2 fields, an int and a long:
 487          // T_INLINE_TYPE T_INT T_INLINE_TYPE T_INT T_LONG T_VOID (second
 488          // slot for the T_LONG) T_VOID (inner T_INLINE_TYPE) T_VOID
 489          // (outer T_INLINE_TYPE)
 490          total_args_passed++;
 491          int vt = 1;
 492          do {
 493            i++;
 494            BasicType bt = sig_extended->at(i)._bt;
 495            BasicType prev_bt = sig_extended->at(i-1)._bt;
 496            if (bt == T_INLINE_TYPE) {
 497              vt++;
 498            } else if (bt == T_VOID &&
 499                       prev_bt != T_LONG &&
 500                       prev_bt != T_DOUBLE) {
 501              vt--;
 502            }
 503          } while (vt != 0);
 504        } else {
 505          total_args_passed++;
 506        }
 507      }
 508   } else {
 509     total_args_passed = sig_extended->length();
 510   }
 511 
 512   return total_args_passed;
 513 }
 514 
 515 
 516 static void gen_c2i_adapter_helper(MacroAssembler* masm,
 517                                    BasicType bt,
 518                                    BasicType prev_bt,
 519                                    size_t size_in_bytes,
 520                                    const VMRegPair& reg_pair,
 521                                    const Address& to,
 522                                    Register tmp1,
 523                                    Register tmp2,
 524                                    Register tmp3,
 525                                    int extraspace,
 526                                    bool is_oop) {
 527   assert(bt != T_INLINE_TYPE || !InlineTypePassFieldsAsArgs, "no inline type here");
 528   if (bt == T_VOID) {
 529     assert(prev_bt == T_LONG || prev_bt == T_DOUBLE, "missing half");
 530     return;
 531   }
 532 
 533   // Say 4 args:
 534   // i   st_off
 535   // 0   32 T_LONG
 536   // 1   24 T_VOID
 537   // 2   16 T_OBJECT
 538   // 3    8 T_BOOL
 539   // -    0 return address
 540   //
 541   // However to make thing extra confusing. Because we can fit a Java long/double in
 542   // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
 543   // leaves one slot empty and only stores to a single slot. In this case the
 544   // slot that is occupied is the T_VOID slot. See I said it was confusing.
 545 
 546   bool wide = (size_in_bytes == wordSize);
 547   VMReg r_1 = reg_pair.first();
 548   VMReg r_2 = reg_pair.second();
 549   assert(r_2->is_valid() == wide, "invalid size");
 550   if (!r_1->is_valid()) {
 551     assert(!r_2->is_valid(), "");
 552     return;
 553   }
 554 
 555   if (!r_1->is_FloatRegister()) {
 556     Register val = tmp3;
 557     if (r_1->is_stack()) {
 558       // memory to memory use tmp3 (scratch registers are used by store_heap_oop)
 559       int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 560       __ load_sized_value(val, Address(sp, ld_off), size_in_bytes, /* is_signed */ false);
 561     } else {
 562       val = r_1->as_Register();
 563     }
 564     assert_different_registers(to.base(), val, rscratch2, tmp1, tmp2);
 565     if (is_oop) {
 566       __ store_heap_oop(to, val, rscratch2, tmp1, tmp2, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
 567     } else {
 568       __ store_sized_value(to, val, size_in_bytes);
 569     }
 570   } else {
 571     if (wide) {
 572       __ strd(r_1->as_FloatRegister(), to);
 573     } else {
 574       // only a float use just part of the slot
 575       __ strs(r_1->as_FloatRegister(), to);
 576     }
 577   }
 578 }
 579 
 580 static void gen_c2i_adapter(MacroAssembler *masm,
 581                             const GrowableArray<SigEntry>* sig_extended,
 582                             const VMRegPair *regs,
 583                             Label& skip_fixup,
 584                             address start,
 585                             OopMapSet* oop_maps,
 586                             int& frame_complete,
 587                             int& frame_size_in_words,
 588                             bool alloc_inline_receiver) {
 589 
 590   // Before we get into the guts of the C2I adapter, see if we should be here
 591   // at all.  We've come from compiled code and are attempting to jump to the
 592   // interpreter, which means the caller made a static call to get here
 593   // (vcalls always get a compiled target if there is one).  Check for a
 594   // compiled target.  If there is one, we need to patch the caller's call.
 595   patch_callers_callsite(masm);
 596 
 597   __ bind(skip_fixup);
 598 
 599   // Name some registers to be used in the following code. We can use
 600   // anything except r0-r7 which are arguments in the Java calling
 601   // convention, rmethod (r12), and r13 which holds the outgoing sender
 602   // SP for the interpreter.
 603   Register buf_array = r10;   // Array of buffered inline types
 604   Register buf_oop = r11;     // Buffered inline type oop
 605   Register tmp1 = r15;
 606   Register tmp2 = r16;
 607   Register tmp3 = r17;
 608 
 609   if (InlineTypePassFieldsAsArgs) {
 610     // Is there an inline type argument?
 611     bool has_inline_argument = false;
 612     for (int i = 0; i < sig_extended->length() && !has_inline_argument; i++) {
 613       has_inline_argument = (sig_extended->at(i)._bt == T_INLINE_TYPE);
 614     }
 615     if (has_inline_argument) {
 616       // There is at least an inline type argument: we're coming from
 617       // compiled code so we have no buffers to back the inline types
 618       // Allocate the buffers here with a runtime call.
 619       RegisterSaver reg_save(false /* save_vectors */);
 620       OopMap* map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
 621 
 622       frame_complete = __ offset();
 623       address the_pc = __ pc();
 624 
 625       Label retaddr;
 626       __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
 627 
 628       __ mov(c_rarg0, rthread);
 629       __ mov(c_rarg1, rmethod);
 630       __ mov(c_rarg2, (int64_t)alloc_inline_receiver);
 631 
 632       __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::allocate_inline_types)));
 633       __ blr(rscratch1);
 634       __ bind(retaddr);
 635 
 636       oop_maps->add_gc_map(__ pc() - start, map);
 637       __ reset_last_Java_frame(false);
 638 
 639       reg_save.restore_live_registers(masm);
 640 
 641       Label no_exception;
 642       __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 643       __ cbz(rscratch1, no_exception);
 644 
 645       __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
 646       __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
 647       __ b(RuntimeAddress(StubRoutines::forward_exception_entry()));
 648 
 649       __ bind(no_exception);
 650 
 651       // We get an array of objects from the runtime call
 652       __ get_vm_result(buf_array, rthread);
 653       __ get_vm_result_2(rmethod, rthread); // TODO: required to keep the callee Method live?
 654     }
 655   }
 656 
 657   // Since all args are passed on the stack, total_args_passed *
 658   // Interpreter::stackElementSize is the space we need.
 659 
 660   int total_args_passed = compute_total_args_passed_int(sig_extended);
 661   int extraspace = total_args_passed * Interpreter::stackElementSize;
 662 
 663   // stack is aligned, keep it that way
 664   extraspace = align_up(extraspace, StackAlignmentInBytes);
 665 
 666   // set senderSP value
 667   __ mov(r13, sp);
 668 
 669   __ sub(sp, sp, extraspace);
 670 
 671   // Now write the args into the outgoing interpreter space
 672 
 673   // next_arg_comp is the next argument from the compiler point of
 674   // view (inline type fields are passed in registers/on the stack). In
 675   // sig_extended, an inline type argument starts with: T_INLINE_TYPE,
 676   // followed by the types of the fields of the inline type and T_VOID
 677   // to mark the end of the inline type. ignored counts the number of
 678   // T_INLINE_TYPE/T_VOID. next_vt_arg is the next inline type argument:
 679   // used to get the buffer for that argument from the pool of buffers
 680   // we allocated above and want to pass to the
 681   // interpreter. next_arg_int is the next argument from the
 682   // interpreter point of view (inline types are passed by reference).
 683   for (int next_arg_comp = 0, ignored = 0, next_vt_arg = 0, next_arg_int = 0;
 684        next_arg_comp < sig_extended->length(); next_arg_comp++) {
 685     assert(ignored <= next_arg_comp, "shouldn't skip over more slots than there are arguments");
 686     assert(next_arg_int <= total_args_passed, "more arguments for the interpreter than expected?");
 687     BasicType bt = sig_extended->at(next_arg_comp)._bt;
 688     int st_off = (total_args_passed - next_arg_int - 1) * Interpreter::stackElementSize;
 689     if (!InlineTypePassFieldsAsArgs || bt != T_INLINE_TYPE) {
 690       int next_off = st_off - Interpreter::stackElementSize;
 691       const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : st_off;
 692       const VMRegPair reg_pair = regs[next_arg_comp-ignored];
 693       size_t size_in_bytes = reg_pair.second()->is_valid() ? 8 : 4;
 694       gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
 695                              size_in_bytes, reg_pair, Address(sp, offset), tmp1, tmp2, tmp3, extraspace, false);
 696       next_arg_int++;
 697 #ifdef ASSERT
 698       if (bt == T_LONG || bt == T_DOUBLE) {
 699         // Overwrite the unused slot with known junk
 700         __ mov(rscratch1, CONST64(0xdeadffffdeadaaaa));
 701         __ str(rscratch1, Address(sp, st_off));
 702       }
 703 #endif /* ASSERT */
 704     } else {
 705       ignored++;
 706       // get the buffer from the just allocated pool of buffers
 707       int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + next_vt_arg * type2aelembytes(T_INLINE_TYPE);
 708       __ load_heap_oop(buf_oop, Address(buf_array, index));
 709       next_vt_arg++; next_arg_int++;
 710       int vt = 1;
 711       // write fields we get from compiled code in registers/stack
 712       // slots to the buffer: we know we are done with that inline type
 713       // argument when we hit the T_VOID that acts as an end of inline
 714       // type delimiter for this inline type. Inline types are flattened
 715       // so we might encounter embedded inline types. Each entry in
 716       // sig_extended contains a field offset in the buffer.
 717       do {
 718         next_arg_comp++;
 719         BasicType bt = sig_extended->at(next_arg_comp)._bt;
 720         BasicType prev_bt = sig_extended->at(next_arg_comp - 1)._bt;
 721         if (bt == T_INLINE_TYPE) {
 722           vt++;
 723           ignored++;
 724         } else if (bt == T_VOID && prev_bt != T_LONG && prev_bt != T_DOUBLE) {
 725           vt--;
 726           ignored++;
 727         } else {
 728           int off = sig_extended->at(next_arg_comp)._offset;
 729           assert(off > 0, "offset in object should be positive");
 730           size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
 731           bool is_oop = is_reference_type(bt);
 732           gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
 733                                  size_in_bytes, regs[next_arg_comp-ignored], Address(buf_oop, off), tmp1, tmp2, tmp3, extraspace, is_oop);
 734         }
 735       } while (vt != 0);
 736       // pass the buffer to the interpreter
 737       __ str(buf_oop, Address(sp, st_off));
 738     }
 739   }
 740 
 741   __ mov(esp, sp); // Interp expects args on caller's expression stack
 742 
 743   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
 744   __ br(rscratch1);
 745 }
 746 
 747 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm, int comp_args_on_stack, const GrowableArray<SigEntry>* sig, const VMRegPair *regs) {
 748 
 749 
 750   // Note: r13 contains the senderSP on entry. We must preserve it since
 751   // we may do a i2c -> c2i transition if we lose a race where compiled
 752   // code goes non-entrant while we get args ready.
 753 
 754   // In addition we use r13 to locate all the interpreter args because
 755   // we must align the stack to 16 bytes.
 756 
 757   // Adapters are frameless.
 758 
 759   // An i2c adapter is frameless because the *caller* frame, which is
 760   // interpreted, routinely repairs its own esp (from
 761   // interpreter_frame_last_sp), even if a callee has modified the
 762   // stack pointer.  It also recalculates and aligns sp.
 763 
 764   // A c2i adapter is frameless because the *callee* frame, which is
 765   // interpreted, routinely repairs its caller's sp (from sender_sp,
 766   // which is set up via the senderSP register).
 767 
 768   // In other words, if *either* the caller or callee is interpreted, we can
 769   // get the stack pointer repaired after a call.
 770 
 771   // This is why c2i and i2c adapters cannot be indefinitely composed.
 772   // In particular, if a c2i adapter were to somehow call an i2c adapter,
 773   // both caller and callee would be compiled methods, and neither would
 774   // clean up the stack pointer changes performed by the two adapters.
 775   // If this happens, control eventually transfers back to the compiled
 776   // caller, but with an uncorrected stack, causing delayed havoc.
 777 
 778   if (VerifyAdapterCalls &&
 779       (Interpreter::code() != NULL || StubRoutines::code1() != NULL)) {
 780 #if 0
 781     // So, let's test for cascading c2i/i2c adapters right now.
 782     //  assert(Interpreter::contains($return_addr) ||
 783     //         StubRoutines::contains($return_addr),
 784     //         "i2c adapter must return to an interpreter frame");
 785     __ block_comment("verify_i2c { ");
 786     Label L_ok;
 787     if (Interpreter::code() != NULL)
 788       range_check(masm, rax, r11,
 789                   Interpreter::code()->code_start(), Interpreter::code()->code_end(),
 790                   L_ok);
 791     if (StubRoutines::code1() != NULL)
 792       range_check(masm, rax, r11,
 793                   StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
 794                   L_ok);
 795     if (StubRoutines::code2() != NULL)
 796       range_check(masm, rax, r11,
 797                   StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
 798                   L_ok);
 799     const char* msg = "i2c adapter must return to an interpreter frame";
 800     __ block_comment(msg);
 801     __ stop(msg);
 802     __ bind(L_ok);
 803     __ block_comment("} verify_i2ce ");
 804 #endif
 805   }
 806 
 807   // Cut-out for having no stack args.
 808   int comp_words_on_stack = 0;
 809   if (comp_args_on_stack) {
 810      comp_words_on_stack = align_up(comp_args_on_stack * VMRegImpl::stack_slot_size, wordSize) >> LogBytesPerWord;
 811      __ sub(rscratch1, sp, comp_words_on_stack * wordSize);
 812      __ andr(sp, rscratch1, -16);
 813   }
 814 
 815   // Will jump to the compiled code just as if compiled code was doing it.
 816   // Pre-load the register-jump target early, to schedule it better.
 817   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_inline_offset())));
 818 
 819 #if INCLUDE_JVMCI
 820   if (EnableJVMCI) {
 821     // check if this call should be routed towards a specific entry point
 822     __ ldr(rscratch2, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 823     Label no_alternative_target;
 824     __ cbz(rscratch2, no_alternative_target);
 825     __ mov(rscratch1, rscratch2);
 826     __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
 827     __ bind(no_alternative_target);
 828   }
 829 #endif // INCLUDE_JVMCI
 830 
 831   int total_args_passed = sig->length();
 832 
 833   // Now generate the shuffle code.
 834   for (int i = 0; i < total_args_passed; i++) {
 835     BasicType bt = sig->at(i)._bt;
 836 
 837     assert(bt != T_INLINE_TYPE, "i2c adapter doesn't unpack inline typ args");
 838     if (bt == T_VOID) {
 839       assert(i > 0 && (sig->at(i - 1)._bt == T_LONG || sig->at(i - 1)._bt == T_DOUBLE), "missing half");
 840       continue;
 841     }
 842 
 843     // Pick up 0, 1 or 2 words from SP+offset.
 844     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(), "scrambled load targets?");
 845 
 846     // Load in argument order going down.
 847     int ld_off = (total_args_passed - i - 1) * Interpreter::stackElementSize;
 848     // Point to interpreter value (vs. tag)
 849     int next_off = ld_off - Interpreter::stackElementSize;
 850     //
 851     //
 852     //
 853     VMReg r_1 = regs[i].first();
 854     VMReg r_2 = regs[i].second();
 855     if (!r_1->is_valid()) {
 856       assert(!r_2->is_valid(), "");
 857       continue;
 858     }
 859     if (r_1->is_stack()) {
 860       // Convert stack slot to an SP offset (+ wordSize to account for return address )
 861       int st_off = regs[i].first()->reg2stack() * VMRegImpl::stack_slot_size;
 862       if (!r_2->is_valid()) {
 863         // sign extend???
 864         __ ldrsw(rscratch2, Address(esp, ld_off));
 865         __ str(rscratch2, Address(sp, st_off));
 866       } else {
 867         //
 868         // We are using two optoregs. This can be either T_OBJECT,
 869         // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 870         // two slots but only uses one for thr T_LONG or T_DOUBLE case
 871         // So we must adjust where to pick up the data to match the
 872         // interpreter.
 873         //
 874         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 875         // are accessed as negative so LSW is at LOW address
 876 
 877         // ld_off is MSW so get LSW
 878         const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;
 879         __ ldr(rscratch2, Address(esp, offset));
 880         // st_off is LSW (i.e. reg.first())
 881          __ str(rscratch2, Address(sp, st_off));
 882        }
 883      } else if (r_1->is_Register()) {  // Register argument
 884        Register r = r_1->as_Register();
 885        if (r_2->is_valid()) {
 886          //
 887          // We are using two VMRegs. This can be either T_OBJECT,
 888          // T_ADDRESS, T_LONG, or T_DOUBLE the interpreter allocates
 889          // two slots but only uses one for thr T_LONG or T_DOUBLE case
 890          // So we must adjust where to pick up the data to match the
 891          // interpreter.
 892 
 893         const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : ld_off;
 894 
 895          // this can be a misaligned move
 896          __ ldr(r, Address(esp, offset));
 897        } else {
 898          // sign extend and use a full word?
 899          __ ldrw(r, Address(esp, ld_off));
 900        }
 901      } else {
 902        if (!r_2->is_valid()) {
 903          __ ldrs(r_1->as_FloatRegister(), Address(esp, ld_off));
 904        } else {
 905          __ ldrd(r_1->as_FloatRegister(), Address(esp, next_off));
 906        }
 907      }
 908    }
 909 
 910 
 911   // 6243940 We might end up in handle_wrong_method if
 912   // the callee is deoptimized as we race thru here. If that
 913   // happens we don't want to take a safepoint because the
 914   // caller frame will look interpreted and arguments are now
 915   // "compiled" so it is much better to make this transition
 916   // invisible to the stack walking code. Unfortunately if
 917   // we try and find the callee by normal means a safepoint
 918   // is possible. So we stash the desired callee in the thread
 919   // and the vm will find there should this case occur.
 920 
 921   __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));
 922   __ br(rscratch1);
 923 }
 924 
 925 static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) {
 926 
 927   Label ok;
 928 
 929   Register holder = rscratch2;
 930   Register receiver = j_rarg0;
 931   Register tmp = r10;  // A call-clobbered register not used for arg passing
 932 
 933   // -------------------------------------------------------------------------
 934   // Generate a C2I adapter.  On entry we know rmethod holds the Method* during calls
 935   // to the interpreter.  The args start out packed in the compiled layout.  They
 936   // need to be unpacked into the interpreter layout.  This will almost always
 937   // require some stack space.  We grow the current (compiled) stack, then repack
 938   // the args.  We  finally end in a jump to the generic interpreter entry point.
 939   // On exit from the interpreter, the interpreter will restore our SP (lest the
 940   // compiled code, which relys solely on SP and not FP, get sick).
 941 
 942   {
 943     __ block_comment("c2i_unverified_entry {");
 944     __ load_klass(rscratch1, receiver);
 945     __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
 946     __ cmp(rscratch1, tmp);
 947     __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
 948     __ br(Assembler::EQ, ok);
 949     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 950 
 951     __ bind(ok);
 952     // Method might have been compiled since the call site was patched to
 953     // interpreted; if that is the case treat it as a miss so we can get
 954     // the call site corrected.
 955     __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 956     __ cbz(rscratch1, skip_fixup);
 957     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 958     __ block_comment("} c2i_unverified_entry");
 959   }
 960 }
 961 
 962 
 963 // ---------------------------------------------------------------
 964 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,
 965                                                             int comp_args_on_stack,
 966                                                             const GrowableArray<SigEntry>* sig,
 967                                                             const VMRegPair* regs,
 968                                                             const GrowableArray<SigEntry>* sig_cc,
 969                                                             const VMRegPair* regs_cc,
 970                                                             const GrowableArray<SigEntry>* sig_cc_ro,
 971                                                             const VMRegPair* regs_cc_ro,
 972                                                             AdapterFingerPrint* fingerprint,
 973                                                             AdapterBlob*& new_adapter,
 974                                                             bool allocate_code_blob) {
 975 
 976   address i2c_entry = __ pc();
 977   gen_i2c_adapter(masm, comp_args_on_stack, sig, regs);
 978 
 979   address c2i_unverified_entry = __ pc();
 980   Label skip_fixup;
 981 
 982   gen_inline_cache_check(masm, skip_fixup);
 983 
 984   OopMapSet* oop_maps = new OopMapSet();
 985   int frame_complete = CodeOffsets::frame_never_safe;
 986   int frame_size_in_words = 0;
 987 
 988   // Scalarized c2i adapter with non-scalarized receiver (i.e., don't pack receiver)
 989   address c2i_inline_ro_entry = __ pc();
 990   if (regs_cc != regs_cc_ro) {
 991     gen_c2i_adapter(masm, sig_cc_ro, regs_cc_ro, skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, false);
 992     skip_fixup.reset();
 993   }
 994 
 995   // Scalarized c2i adapter
 996   address c2i_entry = __ pc();
 997 
 998   // Class initialization barrier for static methods
 999   address c2i_no_clinit_check_entry = NULL;
1000   if (VM_Version::supports_fast_class_init_checks()) {
1001     Label L_skip_barrier;
1002 
1003     { // Bypass the barrier for non-static methods
1004       __ ldrw(rscratch1, Address(rmethod, Method::access_flags_offset()));
1005       __ andsw(zr, rscratch1, JVM_ACC_STATIC);
1006       __ br(Assembler::EQ, L_skip_barrier); // non-static
1007     }
1008 
1009     __ load_method_holder(rscratch2, rmethod);
1010     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1011     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1012 
1013     __ bind(L_skip_barrier);
1014     c2i_no_clinit_check_entry = __ pc();
1015   }
1016 
1017   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1018   bs->c2i_entry_barrier(masm);
1019 
1020   gen_c2i_adapter(masm, sig_cc, regs_cc, skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, true);
1021 
1022   address c2i_unverified_inline_entry = c2i_unverified_entry;
1023 
1024   // Non-scalarized c2i adapter
1025   address c2i_inline_entry = c2i_entry;
1026   if (regs != regs_cc) {
1027     Label inline_entry_skip_fixup;
1028     c2i_unverified_inline_entry = __ pc();
1029     gen_inline_cache_check(masm, inline_entry_skip_fixup);
1030 
1031     c2i_inline_entry = __ pc();
1032     gen_c2i_adapter(masm, sig, regs, inline_entry_skip_fixup, i2c_entry, oop_maps, frame_complete, frame_size_in_words, false);
1033   }
1034 
1035   __ flush();
1036 
1037   // The c2i adapter might safepoint and trigger a GC. The caller must make sure that
1038   // the GC knows about the location of oop argument locations passed to the c2i adapter.
1039   if (allocate_code_blob) {
1040     bool caller_must_gc_arguments = (regs != regs_cc);
1041     new_adapter = AdapterBlob::create(masm->code(), frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
1042   }
1043 
1044   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_inline_entry, c2i_inline_ro_entry, c2i_unverified_entry, c2i_unverified_inline_entry, c2i_no_clinit_check_entry);
1045 }
1046 
1047 static int c_calling_convention_priv(const BasicType *sig_bt,
1048                                          VMRegPair *regs,
1049                                          VMRegPair *regs2,
1050                                          int total_args_passed) {
1051   assert(regs2 == NULL, "not needed on AArch64");
1052 
1053 // We return the amount of VMRegImpl stack slots we need to reserve for all
1054 // the arguments NOT counting out_preserve_stack_slots.
1055 
1056     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1057       c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5,  c_rarg6,  c_rarg7
1058     };
1059     static const FloatRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
1060       c_farg0, c_farg1, c_farg2, c_farg3,
1061       c_farg4, c_farg5, c_farg6, c_farg7
1062     };
1063 
1064     uint int_args = 0;
1065     uint fp_args = 0;
1066     uint stk_args = 0; // inc by 2 each time
1067 
1068     for (int i = 0; i < total_args_passed; i++) {
1069       switch (sig_bt[i]) {
1070       case T_BOOLEAN:
1071       case T_CHAR:
1072       case T_BYTE:
1073       case T_SHORT:
1074       case T_INT:
1075         if (int_args < Argument::n_int_register_parameters_c) {
1076           regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
1077         } else {
1078 #ifdef __APPLE__
1079           // Less-than word types are stored one after another.
1080           // The code is unable to handle this so bailout.
1081           return -1;
1082 #endif
1083           regs[i].set1(VMRegImpl::stack2reg(stk_args));
1084           stk_args += 2;
1085         }
1086         break;
1087       case T_LONG:
1088         assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
1089         // fall through
1090       case T_OBJECT:
1091       case T_ARRAY:
1092       case T_INLINE_TYPE:
1093       case T_ADDRESS:
1094       case T_METADATA:
1095         if (int_args < Argument::n_int_register_parameters_c) {
1096           regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
1097         } else {
1098           regs[i].set2(VMRegImpl::stack2reg(stk_args));
1099           stk_args += 2;
1100         }
1101         break;
1102       case T_FLOAT:
1103         if (fp_args < Argument::n_float_register_parameters_c) {
1104           regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
1105         } else {
1106 #ifdef __APPLE__
1107           // Less-than word types are stored one after another.
1108           // The code is unable to handle this so bailout.
1109           return -1;
1110 #endif
1111           regs[i].set1(VMRegImpl::stack2reg(stk_args));
1112           stk_args += 2;
1113         }
1114         break;
1115       case T_DOUBLE:
1116         assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
1117         if (fp_args < Argument::n_float_register_parameters_c) {
1118           regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
1119         } else {
1120           regs[i].set2(VMRegImpl::stack2reg(stk_args));
1121           stk_args += 2;
1122         }
1123         break;
1124       case T_VOID: // Halves of longs and doubles
1125         assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
1126         regs[i].set_bad();
1127         break;
1128       default:
1129         ShouldNotReachHere();
1130         break;
1131       }
1132     }
1133 
1134   return stk_args;
1135 }
1136 
1137 int SharedRuntime::vector_calling_convention(VMRegPair *regs,
1138                                              uint num_bits,
1139                                              uint total_args_passed) {
1140   Unimplemented();
1141   return 0;
1142 }
1143 
1144 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
1145                                          VMRegPair *regs,
1146                                          VMRegPair *regs2,
1147                                          int total_args_passed)
1148 {
1149   int result = c_calling_convention_priv(sig_bt, regs, regs2, total_args_passed);
1150   guarantee(result >= 0, "Unsupported arguments configuration");
1151   return result;
1152 }
1153 
1154 // On 64 bit we will store integer like items to the stack as
1155 // 64 bits items (Aarch64 abi) even though java would only store
1156 // 32bits for a parameter. On 32bit it will simply be 32 bits
1157 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
1158 static void move32_64(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1159   if (src.first()->is_stack()) {
1160     if (dst.first()->is_stack()) {
1161       // stack to stack
1162       __ ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
1163       __ str(rscratch1, Address(sp, reg2offset_out(dst.first())));
1164     } else {
1165       // stack to reg
1166       __ ldrsw(dst.first()->as_Register(), Address(rfp, reg2offset_in(src.first())));
1167     }
1168   } else if (dst.first()->is_stack()) {
1169     // reg to stack
1170     // Do we really have to sign extend???
1171     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
1172     __ str(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first())));
1173   } else {
1174     if (dst.first() != src.first()) {
1175       __ sxtw(dst.first()->as_Register(), src.first()->as_Register());
1176     }
1177   }
1178 }
1179 
1180 // An oop arg. Must pass a handle not the oop itself
1181 static void object_move(MacroAssembler* masm,
1182                         OopMap* map,
1183                         int oop_handle_offset,
1184                         int framesize_in_slots,
1185                         VMRegPair src,
1186                         VMRegPair dst,
1187                         bool is_receiver,
1188                         int* receiver_offset) {
1189 
1190   // must pass a handle. First figure out the location we use as a handle
1191 
1192   Register rHandle = dst.first()->is_stack() ? rscratch2 : dst.first()->as_Register();
1193 
1194   // See if oop is NULL if it is we need no handle
1195 
1196   if (src.first()->is_stack()) {
1197 
1198     // Oop is already on the stack as an argument
1199     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
1200     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
1201     if (is_receiver) {
1202       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
1203     }
1204 
1205     __ ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
1206     __ lea(rHandle, Address(rfp, reg2offset_in(src.first())));
1207     // conditionally move a NULL
1208     __ cmp(rscratch1, zr);
1209     __ csel(rHandle, zr, rHandle, Assembler::EQ);
1210   } else {
1211 
1212     // Oop is in an a register we must store it to the space we reserve
1213     // on the stack for oop_handles and pass a handle if oop is non-NULL
1214 
1215     const Register rOop = src.first()->as_Register();
1216     int oop_slot;
1217     if (rOop == j_rarg0)
1218       oop_slot = 0;
1219     else if (rOop == j_rarg1)
1220       oop_slot = 1;
1221     else if (rOop == j_rarg2)
1222       oop_slot = 2;
1223     else if (rOop == j_rarg3)
1224       oop_slot = 3;
1225     else if (rOop == j_rarg4)
1226       oop_slot = 4;
1227     else if (rOop == j_rarg5)
1228       oop_slot = 5;
1229     else if (rOop == j_rarg6)
1230       oop_slot = 6;
1231     else {
1232       assert(rOop == j_rarg7, "wrong register");
1233       oop_slot = 7;
1234     }
1235 
1236     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
1237     int offset = oop_slot*VMRegImpl::stack_slot_size;
1238 
1239     map->set_oop(VMRegImpl::stack2reg(oop_slot));
1240     // Store oop in handle area, may be NULL
1241     __ str(rOop, Address(sp, offset));
1242     if (is_receiver) {
1243       *receiver_offset = offset;
1244     }
1245 
1246     __ cmp(rOop, zr);
1247     __ lea(rHandle, Address(sp, offset));
1248     // conditionally move a NULL
1249     __ csel(rHandle, zr, rHandle, Assembler::EQ);
1250   }
1251 
1252   // If arg is on the stack then place it otherwise it is already in correct reg.
1253   if (dst.first()->is_stack()) {
1254     __ str(rHandle, Address(sp, reg2offset_out(dst.first())));
1255   }
1256 }
1257 
1258 // A float arg may have to do float reg int reg conversion
1259 static void float_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1260   assert(src.first()->is_stack() && dst.first()->is_stack() ||
1261          src.first()->is_reg() && dst.first()->is_reg(), "Unexpected error");
1262   if (src.first()->is_stack()) {
1263     if (dst.first()->is_stack()) {
1264       __ ldrw(rscratch1, Address(rfp, reg2offset_in(src.first())));
1265       __ strw(rscratch1, Address(sp, reg2offset_out(dst.first())));
1266     } else {
1267       ShouldNotReachHere();
1268     }
1269   } else if (src.first() != dst.first()) {
1270     if (src.is_single_phys_reg() && dst.is_single_phys_reg())
1271       __ fmovs(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
1272     else
1273       ShouldNotReachHere();
1274   }
1275 }
1276 
1277 // A long move
1278 static void long_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1279   if (src.first()->is_stack()) {
1280     if (dst.first()->is_stack()) {
1281       // stack to stack
1282       __ ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
1283       __ str(rscratch1, Address(sp, reg2offset_out(dst.first())));
1284     } else {
1285       // stack to reg
1286       __ ldr(dst.first()->as_Register(), Address(rfp, reg2offset_in(src.first())));
1287     }
1288   } else if (dst.first()->is_stack()) {
1289     // reg to stack
1290     // Do we really have to sign extend???
1291     // __ movslq(src.first()->as_Register(), src.first()->as_Register());
1292     __ str(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first())));
1293   } else {
1294     if (dst.first() != src.first()) {
1295       __ mov(dst.first()->as_Register(), src.first()->as_Register());
1296     }
1297   }
1298 }
1299 
1300 
1301 // A double move
1302 static void double_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1303   assert(src.first()->is_stack() && dst.first()->is_stack() ||
1304          src.first()->is_reg() && dst.first()->is_reg(), "Unexpected error");
1305   if (src.first()->is_stack()) {
1306     if (dst.first()->is_stack()) {
1307       __ ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
1308       __ str(rscratch1, Address(sp, reg2offset_out(dst.first())));
1309     } else {
1310       ShouldNotReachHere();
1311     }
1312   } else if (src.first() != dst.first()) {
1313     if (src.is_single_phys_reg() && dst.is_single_phys_reg())
1314       __ fmovd(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
1315     else
1316       ShouldNotReachHere();
1317   }
1318 }
1319 
1320 
1321 void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1322   // We always ignore the frame_slots arg and just use the space just below frame pointer
1323   // which by this time is free to use
1324   switch (ret_type) {
1325   case T_FLOAT:
1326     __ strs(v0, Address(rfp, -wordSize));
1327     break;
1328   case T_DOUBLE:
1329     __ strd(v0, Address(rfp, -wordSize));
1330     break;
1331   case T_VOID:  break;
1332   default: {
1333     __ str(r0, Address(rfp, -wordSize));
1334     }
1335   }
1336 }
1337 
1338 void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1339   // We always ignore the frame_slots arg and just use the space just below frame pointer
1340   // which by this time is free to use
1341   switch (ret_type) {
1342   case T_FLOAT:
1343     __ ldrs(v0, Address(rfp, -wordSize));
1344     break;
1345   case T_DOUBLE:
1346     __ ldrd(v0, Address(rfp, -wordSize));
1347     break;
1348   case T_VOID:  break;
1349   default: {
1350     __ ldr(r0, Address(rfp, -wordSize));
1351     }
1352   }
1353 }
1354 static void save_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
1355   RegSet x;
1356   for ( int i = first_arg ; i < arg_count ; i++ ) {
1357     if (args[i].first()->is_Register()) {
1358       x = x + args[i].first()->as_Register();
1359     } else if (args[i].first()->is_FloatRegister()) {
1360       __ strd(args[i].first()->as_FloatRegister(), Address(__ pre(sp, -2 * wordSize)));
1361     }
1362   }
1363   __ push(x, sp);
1364 }
1365 
1366 static void restore_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
1367   RegSet x;
1368   for ( int i = first_arg ; i < arg_count ; i++ ) {
1369     if (args[i].first()->is_Register()) {
1370       x = x + args[i].first()->as_Register();
1371     } else {
1372       ;
1373     }
1374   }
1375   __ pop(x, sp);
1376   for ( int i = arg_count - 1 ; i >= first_arg ; i-- ) {
1377     if (args[i].first()->is_Register()) {
1378       ;
1379     } else if (args[i].first()->is_FloatRegister()) {
1380       __ ldrd(args[i].first()->as_FloatRegister(), Address(__ post(sp, 2 * wordSize)));
1381     }
1382   }
1383 }
1384 
1385 // Unpack an array argument into a pointer to the body and the length
1386 // if the array is non-null, otherwise pass 0 for both.
1387 static void unpack_array_argument(MacroAssembler* masm, VMRegPair reg, BasicType in_elem_type, VMRegPair body_arg, VMRegPair length_arg) { Unimplemented(); }
1388 
1389 
1390 class ComputeMoveOrder: public StackObj {
1391   class MoveOperation: public ResourceObj {
1392     friend class ComputeMoveOrder;
1393    private:
1394     VMRegPair        _src;
1395     VMRegPair        _dst;
1396     int              _src_index;
1397     int              _dst_index;
1398     bool             _processed;
1399     MoveOperation*  _next;
1400     MoveOperation*  _prev;
1401 
1402     static int get_id(VMRegPair r) { Unimplemented(); return 0; }
1403 
1404    public:
1405     MoveOperation(int src_index, VMRegPair src, int dst_index, VMRegPair dst):
1406       _src(src)
1407     , _dst(dst)
1408     , _src_index(src_index)
1409     , _dst_index(dst_index)
1410     , _processed(false)
1411     , _next(NULL)
1412     , _prev(NULL) { Unimplemented(); }
1413 
1414     VMRegPair src() const              { Unimplemented(); return _src; }
1415     int src_id() const                 { Unimplemented(); return 0; }
1416     int src_index() const              { Unimplemented(); return 0; }
1417     VMRegPair dst() const              { Unimplemented(); return _src; }
1418     void set_dst(int i, VMRegPair dst) { Unimplemented(); }
1419     int dst_index() const              { Unimplemented(); return 0; }
1420     int dst_id() const                 { Unimplemented(); return 0; }
1421     MoveOperation* next() const        { Unimplemented(); return 0; }
1422     MoveOperation* prev() const        { Unimplemented(); return 0; }
1423     void set_processed()               { Unimplemented(); }
1424     bool is_processed() const          { Unimplemented(); return 0; }
1425 
1426     // insert
1427     void break_cycle(VMRegPair temp_register) { Unimplemented(); }
1428 
1429     void link(GrowableArray<MoveOperation*>& killer) { Unimplemented(); }
1430   };
1431 
1432  private:
1433   GrowableArray<MoveOperation*> edges;
1434 
1435  public:
1436   ComputeMoveOrder(int total_in_args, VMRegPair* in_regs, int total_c_args, VMRegPair* out_regs,
1437                     BasicType* in_sig_bt, GrowableArray<int>& arg_order, VMRegPair tmp_vmreg) { Unimplemented(); }
1438 
1439   // Collected all the move operations
1440   void add_edge(int src_index, VMRegPair src, int dst_index, VMRegPair dst) { Unimplemented(); }
1441 
1442   // Walk the edges breaking cycles between moves.  The result list
1443   // can be walked in order to produce the proper set of loads
1444   GrowableArray<MoveOperation*>* get_store_order(VMRegPair temp_register) { Unimplemented(); return 0; }
1445 };
1446 
1447 
1448 static void rt_call(MacroAssembler* masm, address dest) {
1449   CodeBlob *cb = CodeCache::find_blob(dest);
1450   if (cb) {
1451     __ far_call(RuntimeAddress(dest));
1452   } else {
1453     __ lea(rscratch1, RuntimeAddress(dest));
1454     __ blr(rscratch1);
1455   }
1456 }
1457 
1458 static void verify_oop_args(MacroAssembler* masm,
1459                             const methodHandle& method,
1460                             const BasicType* sig_bt,
1461                             const VMRegPair* regs) {
1462   Register temp_reg = r19;  // not part of any compiled calling seq
1463   if (VerifyOops) {
1464     for (int i = 0; i < method->size_of_parameters(); i++) {
1465       if (sig_bt[i] == T_OBJECT ||
1466           sig_bt[i] == T_ARRAY) {
1467         VMReg r = regs[i].first();
1468         assert(r->is_valid(), "bad oop arg");
1469         if (r->is_stack()) {
1470           __ ldr(temp_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size));
1471           __ verify_oop(temp_reg);
1472         } else {
1473           __ verify_oop(r->as_Register());
1474         }
1475       }
1476     }
1477   }
1478 }
1479 
1480 static void gen_special_dispatch(MacroAssembler* masm,
1481                                  const methodHandle& method,
1482                                  const BasicType* sig_bt,
1483                                  const VMRegPair* regs) {
1484   verify_oop_args(masm, method, sig_bt, regs);
1485   vmIntrinsics::ID iid = method->intrinsic_id();
1486 
1487   // Now write the args into the outgoing interpreter space
1488   bool     has_receiver   = false;
1489   Register receiver_reg   = noreg;
1490   int      member_arg_pos = -1;
1491   Register member_reg     = noreg;
1492   int      ref_kind       = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
1493   if (ref_kind != 0) {
1494     member_arg_pos = method->size_of_parameters() - 1;  // trailing MemberName argument
1495     member_reg = r19;  // known to be free at this point
1496     has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
1497   } else if (iid == vmIntrinsics::_invokeBasic || iid == vmIntrinsics::_linkToNative) {
1498     has_receiver = true;
1499   } else {
1500     fatal("unexpected intrinsic id %d", vmIntrinsics::as_int(iid));
1501   }
1502 
1503   if (member_reg != noreg) {
1504     // Load the member_arg into register, if necessary.
1505     SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
1506     VMReg r = regs[member_arg_pos].first();
1507     if (r->is_stack()) {
1508       __ ldr(member_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size));
1509     } else {
1510       // no data motion is needed
1511       member_reg = r->as_Register();
1512     }
1513   }
1514 
1515   if (has_receiver) {
1516     // Make sure the receiver is loaded into a register.
1517     assert(method->size_of_parameters() > 0, "oob");
1518     assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
1519     VMReg r = regs[0].first();
1520     assert(r->is_valid(), "bad receiver arg");
1521     if (r->is_stack()) {
1522       // Porting note:  This assumes that compiled calling conventions always
1523       // pass the receiver oop in a register.  If this is not true on some
1524       // platform, pick a temp and load the receiver from stack.
1525       fatal("receiver always in a register");
1526       receiver_reg = r2;  // known to be free at this point
1527       __ ldr(receiver_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size));
1528     } else {
1529       // no data motion is needed
1530       receiver_reg = r->as_Register();
1531     }
1532   }
1533 
1534   // Figure out which address we are really jumping to:
1535   MethodHandles::generate_method_handle_dispatch(masm, iid,
1536                                                  receiver_reg, member_reg, /*for_compiler_entry:*/ true);
1537 }
1538 
1539 // ---------------------------------------------------------------------------
1540 // Generate a native wrapper for a given method.  The method takes arguments
1541 // in the Java compiled code convention, marshals them to the native
1542 // convention (handlizes oops, etc), transitions to native, makes the call,
1543 // returns to java state (possibly blocking), unhandlizes any result and
1544 // returns.
1545 //
1546 // Critical native functions are a shorthand for the use of
1547 // GetPrimtiveArrayCritical and disallow the use of any other JNI
1548 // functions.  The wrapper is expected to unpack the arguments before
1549 // passing them to the callee. Critical native functions leave the state _in_Java,
1550 // since they block out GC.
1551 // Some other parts of JNI setup are skipped like the tear down of the JNI handle
1552 // block and the check for pending exceptions it's impossible for them
1553 // to be thrown.
1554 //
1555 nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
1556                                                 const methodHandle& method,
1557                                                 int compile_id,
1558                                                 BasicType* in_sig_bt,
1559                                                 VMRegPair* in_regs,
1560                                                 BasicType ret_type,
1561                                                 address critical_entry) {
1562   if (method->is_method_handle_intrinsic()) {
1563     vmIntrinsics::ID iid = method->intrinsic_id();
1564     intptr_t start = (intptr_t)__ pc();
1565     int vep_offset = ((intptr_t)__ pc()) - start;
1566 
1567     // First instruction must be a nop as it may need to be patched on deoptimisation
1568     __ nop();
1569     gen_special_dispatch(masm,
1570                          method,
1571                          in_sig_bt,
1572                          in_regs);
1573     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
1574     __ flush();
1575     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
1576     return nmethod::new_native_nmethod(method,
1577                                        compile_id,
1578                                        masm->code(),
1579                                        vep_offset,
1580                                        frame_complete,
1581                                        stack_slots / VMRegImpl::slots_per_word,
1582                                        in_ByteSize(-1),
1583                                        in_ByteSize(-1),
1584                                        (OopMapSet*)NULL);
1585   }
1586   bool is_critical_native = true;
1587   address native_func = critical_entry;
1588   if (native_func == NULL) {
1589     native_func = method->native_function();
1590     is_critical_native = false;
1591   }
1592   assert(native_func != NULL, "must have function");
1593 
1594   // An OopMap for lock (and class if static)
1595   OopMapSet *oop_maps = new OopMapSet();
1596   intptr_t start = (intptr_t)__ pc();
1597 
1598   // We have received a description of where all the java arg are located
1599   // on entry to the wrapper. We need to convert these args to where
1600   // the jni function will expect them. To figure out where they go
1601   // we convert the java signature to a C signature by inserting
1602   // the hidden arguments as arg[0] and possibly arg[1] (static method)
1603 
1604   const int total_in_args = method->size_of_parameters();
1605   int total_c_args = total_in_args;
1606   if (!is_critical_native) {
1607     total_c_args += 1;
1608     if (method->is_static()) {
1609       total_c_args++;
1610     }
1611   } else {
1612     for (int i = 0; i < total_in_args; i++) {
1613       if (in_sig_bt[i] == T_ARRAY) {
1614         total_c_args++;
1615       }
1616     }
1617   }
1618 
1619   BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
1620   VMRegPair* out_regs   = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
1621   BasicType* in_elem_bt = NULL;
1622 
1623   int argc = 0;
1624   if (!is_critical_native) {
1625     out_sig_bt[argc++] = T_ADDRESS;
1626     if (method->is_static()) {
1627       out_sig_bt[argc++] = T_OBJECT;
1628     }
1629 
1630     for (int i = 0; i < total_in_args ; i++ ) {
1631       out_sig_bt[argc++] = in_sig_bt[i];
1632     }
1633   } else {
1634     in_elem_bt = NEW_RESOURCE_ARRAY(BasicType, total_in_args);
1635     SignatureStream ss(method->signature());
1636     for (int i = 0; i < total_in_args ; i++ ) {
1637       if (in_sig_bt[i] == T_ARRAY) {
1638         // Arrays are passed as int, elem* pair
1639         out_sig_bt[argc++] = T_INT;
1640         out_sig_bt[argc++] = T_ADDRESS;
1641         ss.skip_array_prefix(1);  // skip one '['
1642         assert(ss.is_primitive(), "primitive type expected");
1643         in_elem_bt[i] = ss.type();
1644       } else {
1645         out_sig_bt[argc++] = in_sig_bt[i];
1646         in_elem_bt[i] = T_VOID;
1647       }
1648       if (in_sig_bt[i] != T_VOID) {
1649         assert(in_sig_bt[i] == ss.type() ||
1650                in_sig_bt[i] == T_ARRAY, "must match");
1651         ss.next();
1652       }
1653     }
1654   }
1655 
1656   // Now figure out where the args must be stored and how much stack space
1657   // they require.
1658   int out_arg_slots;
1659   out_arg_slots = c_calling_convention_priv(out_sig_bt, out_regs, NULL, total_c_args);
1660 
1661   if (out_arg_slots < 0) {
1662     return NULL;
1663   }
1664 
1665   // Compute framesize for the wrapper.  We need to handlize all oops in
1666   // incoming registers
1667 
1668   // Calculate the total number of stack slots we will need.
1669 
1670   // First count the abi requirement plus all of the outgoing args
1671   int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots;
1672 
1673   // Now the space for the inbound oop handle area
1674   int total_save_slots = 8 * VMRegImpl::slots_per_word;  // 8 arguments passed in registers
1675   if (is_critical_native) {
1676     // Critical natives may have to call out so they need a save area
1677     // for register arguments.
1678     int double_slots = 0;
1679     int single_slots = 0;
1680     for ( int i = 0; i < total_in_args; i++) {
1681       if (in_regs[i].first()->is_Register()) {
1682         const Register reg = in_regs[i].first()->as_Register();
1683         switch (in_sig_bt[i]) {
1684           case T_BOOLEAN:
1685           case T_BYTE:
1686           case T_SHORT:
1687           case T_CHAR:
1688           case T_INT:  single_slots++; break;
1689           case T_ARRAY:  // specific to LP64 (7145024)
1690           case T_LONG: double_slots++; break;
1691           default:  ShouldNotReachHere();
1692         }
1693       } else if (in_regs[i].first()->is_FloatRegister()) {
1694         ShouldNotReachHere();
1695       }
1696     }
1697     total_save_slots = double_slots * 2 + single_slots;
1698     // align the save area
1699     if (double_slots != 0) {
1700       stack_slots = align_up(stack_slots, 2);
1701     }
1702   }
1703 
1704   int oop_handle_offset = stack_slots;
1705   stack_slots += total_save_slots;
1706 
1707   // Now any space we need for handlizing a klass if static method
1708 
1709   int klass_slot_offset = 0;
1710   int klass_offset = -1;
1711   int lock_slot_offset = 0;
1712   bool is_static = false;
1713 
1714   if (method->is_static()) {
1715     klass_slot_offset = stack_slots;
1716     stack_slots += VMRegImpl::slots_per_word;
1717     klass_offset = klass_slot_offset * VMRegImpl::stack_slot_size;
1718     is_static = true;
1719   }
1720 
1721   // Plus a lock if needed
1722 
1723   if (method->is_synchronized()) {
1724     lock_slot_offset = stack_slots;
1725     stack_slots += VMRegImpl::slots_per_word;
1726   }
1727 
1728   // Now a place (+2) to save return values or temp during shuffling
1729   // + 4 for return address (which we own) and saved rfp
1730   stack_slots += 6;
1731 
1732   // Ok The space we have allocated will look like:
1733   //
1734   //
1735   // FP-> |                     |
1736   //      |---------------------|
1737   //      | 2 slots for moves   |
1738   //      |---------------------|
1739   //      | lock box (if sync)  |
1740   //      |---------------------| <- lock_slot_offset
1741   //      | klass (if static)   |
1742   //      |---------------------| <- klass_slot_offset
1743   //      | oopHandle area      |
1744   //      |---------------------| <- oop_handle_offset (8 java arg registers)
1745   //      | outbound memory     |
1746   //      | based arguments     |
1747   //      |                     |
1748   //      |---------------------|
1749   //      |                     |
1750   // SP-> | out_preserved_slots |
1751   //
1752   //
1753 
1754 
1755   // Now compute actual number of stack words we need rounding to make
1756   // stack properly aligned.
1757   stack_slots = align_up(stack_slots, StackAlignmentInSlots);
1758 
1759   int stack_size = stack_slots * VMRegImpl::stack_slot_size;
1760 
1761   // First thing make an ic check to see if we should even be here
1762 
1763   // We are free to use all registers as temps without saving them and
1764   // restoring them except rfp. rfp is the only callee save register
1765   // as far as the interpreter and the compiler(s) are concerned.
1766 
1767 
1768   const Register ic_reg = rscratch2;
1769   const Register receiver = j_rarg0;
1770 
1771   Label hit;
1772   Label exception_pending;
1773 
1774   assert_different_registers(ic_reg, receiver, rscratch1);
1775   __ verify_oop(receiver);
1776   __ cmp_klass(receiver, ic_reg, rscratch1);
1777   __ br(Assembler::EQ, hit);
1778 
1779   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1780 
1781   // Verified entry point must be aligned
1782   __ align(8);
1783 
1784   __ bind(hit);
1785 
1786   int vep_offset = ((intptr_t)__ pc()) - start;
1787 
1788   // If we have to make this method not-entrant we'll overwrite its
1789   // first instruction with a jump.  For this action to be legal we
1790   // must ensure that this first instruction is a B, BL, NOP, BKPT,
1791   // SVC, HVC, or SMC.  Make it a NOP.
1792   __ nop();
1793 
1794   if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
1795     Label L_skip_barrier;
1796     __ mov_metadata(rscratch2, method->method_holder()); // InstanceKlass*
1797     __ clinit_barrier(rscratch2, rscratch1, &L_skip_barrier);
1798     __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub()));
1799 
1800     __ bind(L_skip_barrier);
1801   }
1802 
1803   // Generate stack overflow check
1804   __ bang_stack_with_offset(checked_cast<int>(StackOverflow::stack_shadow_zone_size()));
1805 
1806   // Generate a new frame for the wrapper.
1807   __ enter();
1808   // -2 because return address is already present and so is saved rfp
1809   __ sub(sp, sp, stack_size - 2*wordSize);
1810 
1811   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
1812   bs->nmethod_entry_barrier(masm);
1813 
1814   // Frame is now completed as far as size and linkage.
1815   int frame_complete = ((intptr_t)__ pc()) - start;
1816 
1817   // We use r20 as the oop handle for the receiver/klass
1818   // It is callee save so it survives the call to native
1819 
1820   const Register oop_handle_reg = r20;
1821 
1822   //
1823   // We immediately shuffle the arguments so that any vm call we have to
1824   // make from here on out (sync slow path, jvmti, etc.) we will have
1825   // captured the oops from our caller and have a valid oopMap for
1826   // them.
1827 
1828   // -----------------
1829   // The Grand Shuffle
1830 
1831   // The Java calling convention is either equal (linux) or denser (win64) than the
1832   // c calling convention. However the because of the jni_env argument the c calling
1833   // convention always has at least one more (and two for static) arguments than Java.
1834   // Therefore if we move the args from java -> c backwards then we will never have
1835   // a register->register conflict and we don't have to build a dependency graph
1836   // and figure out how to break any cycles.
1837   //
1838 
1839   // Record esp-based slot for receiver on stack for non-static methods
1840   int receiver_offset = -1;
1841 
1842   // This is a trick. We double the stack slots so we can claim
1843   // the oops in the caller's frame. Since we are sure to have
1844   // more args than the caller doubling is enough to make
1845   // sure we can capture all the incoming oop args from the
1846   // caller.
1847   //
1848   OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1849 
1850   // Mark location of rfp (someday)
1851   // map->set_callee_saved(VMRegImpl::stack2reg( stack_slots - 2), stack_slots * 2, 0, vmreg(rfp));
1852 
1853 
1854   int float_args = 0;
1855   int int_args = 0;
1856 
1857 #ifdef ASSERT
1858   bool reg_destroyed[RegisterImpl::number_of_registers];
1859   bool freg_destroyed[FloatRegisterImpl::number_of_registers];
1860   for ( int r = 0 ; r < RegisterImpl::number_of_registers ; r++ ) {
1861     reg_destroyed[r] = false;
1862   }
1863   for ( int f = 0 ; f < FloatRegisterImpl::number_of_registers ; f++ ) {
1864     freg_destroyed[f] = false;
1865   }
1866 
1867 #endif /* ASSERT */
1868 
1869   // This may iterate in two different directions depending on the
1870   // kind of native it is.  The reason is that for regular JNI natives
1871   // the incoming and outgoing registers are offset upwards and for
1872   // critical natives they are offset down.
1873   GrowableArray<int> arg_order(2 * total_in_args);
1874   VMRegPair tmp_vmreg;
1875   tmp_vmreg.set2(r19->as_VMReg());
1876 
1877   if (!is_critical_native) {
1878     for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
1879       arg_order.push(i);
1880       arg_order.push(c_arg);
1881     }
1882   } else {
1883     // Compute a valid move order, using tmp_vmreg to break any cycles
1884     ComputeMoveOrder cmo(total_in_args, in_regs, total_c_args, out_regs, in_sig_bt, arg_order, tmp_vmreg);
1885   }
1886 
1887   int temploc = -1;
1888   for (int ai = 0; ai < arg_order.length(); ai += 2) {
1889     int i = arg_order.at(ai);
1890     int c_arg = arg_order.at(ai + 1);
1891     __ block_comment(err_msg("move %d -> %d", i, c_arg));
1892     if (c_arg == -1) {
1893       assert(is_critical_native, "should only be required for critical natives");
1894       // This arg needs to be moved to a temporary
1895       __ mov(tmp_vmreg.first()->as_Register(), in_regs[i].first()->as_Register());
1896       in_regs[i] = tmp_vmreg;
1897       temploc = i;
1898       continue;
1899     } else if (i == -1) {
1900       assert(is_critical_native, "should only be required for critical natives");
1901       // Read from the temporary location
1902       assert(temploc != -1, "must be valid");
1903       i = temploc;
1904       temploc = -1;
1905     }
1906 #ifdef ASSERT
1907     if (in_regs[i].first()->is_Register()) {
1908       assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
1909     } else if (in_regs[i].first()->is_FloatRegister()) {
1910       assert(!freg_destroyed[in_regs[i].first()->as_FloatRegister()->encoding()], "destroyed reg!");
1911     }
1912     if (out_regs[c_arg].first()->is_Register()) {
1913       reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1914     } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1915       freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1916     }
1917 #endif /* ASSERT */
1918     switch (in_sig_bt[i]) {
1919       case T_ARRAY:
1920         if (is_critical_native) {
1921           unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg]);
1922           c_arg++;
1923 #ifdef ASSERT
1924           if (out_regs[c_arg].first()->is_Register()) {
1925             reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1926           } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1927             freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1928           }
1929 #endif
1930           int_args++;
1931           break;
1932         }
1933       case T_INLINE_TYPE:
1934       case T_OBJECT:
1935         assert(!is_critical_native, "no oop arguments");
1936         object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
1937                     ((i == 0) && (!is_static)),
1938                     &receiver_offset);
1939         int_args++;
1940         break;
1941       case T_VOID:
1942         break;
1943 
1944       case T_FLOAT:
1945         float_move(masm, in_regs[i], out_regs[c_arg]);
1946         float_args++;
1947         break;
1948 
1949       case T_DOUBLE:
1950         assert( i + 1 < total_in_args &&
1951                 in_sig_bt[i + 1] == T_VOID &&
1952                 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
1953         double_move(masm, in_regs[i], out_regs[c_arg]);
1954         float_args++;
1955         break;
1956 
1957       case T_LONG :
1958         long_move(masm, in_regs[i], out_regs[c_arg]);
1959         int_args++;
1960         break;
1961 
1962       case T_ADDRESS: assert(false, "found T_ADDRESS in java args");
1963 
1964       default:
1965         move32_64(masm, in_regs[i], out_regs[c_arg]);
1966         int_args++;
1967     }
1968   }
1969 
1970   // point c_arg at the first arg that is already loaded in case we
1971   // need to spill before we call out
1972   int c_arg = total_c_args - total_in_args;
1973 
1974   // Pre-load a static method's oop into c_rarg1.
1975   if (method->is_static() && !is_critical_native) {
1976 
1977     //  load oop into a register
1978     __ movoop(c_rarg1,
1979               JNIHandles::make_local(method->method_holder()->java_mirror()),
1980               /*immediate*/true);
1981 
1982     // Now handlize the static class mirror it's known not-null.
1983     __ str(c_rarg1, Address(sp, klass_offset));
1984     map->set_oop(VMRegImpl::stack2reg(klass_slot_offset));
1985 
1986     // Now get the handle
1987     __ lea(c_rarg1, Address(sp, klass_offset));
1988     // and protect the arg if we must spill
1989     c_arg--;
1990   }
1991 
1992   // Change state to native (we save the return address in the thread, since it might not
1993   // be pushed on the stack when we do a stack traversal).
1994   // We use the same pc/oopMap repeatedly when we call out
1995 
1996   Label native_return;
1997   __ set_last_Java_frame(sp, noreg, native_return, rscratch1);
1998 
1999   Label dtrace_method_entry, dtrace_method_entry_done;
2000   {
2001     uint64_t offset;
2002     __ adrp(rscratch1, ExternalAddress((address)&DTraceMethodProbes), offset);
2003     __ ldrb(rscratch1, Address(rscratch1, offset));
2004     __ cbnzw(rscratch1, dtrace_method_entry);
2005     __ bind(dtrace_method_entry_done);
2006   }
2007 
2008   // RedefineClasses() tracing support for obsolete method entry
2009   if (log_is_enabled(Trace, redefine, class, obsolete)) {
2010     // protect the args we've loaded
2011     save_args(masm, total_c_args, c_arg, out_regs);
2012     __ mov_metadata(c_rarg1, method());
2013     __ call_VM_leaf(
2014       CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
2015       rthread, c_rarg1);
2016     restore_args(masm, total_c_args, c_arg, out_regs);
2017   }
2018 
2019   // Lock a synchronized method
2020 
2021   // Register definitions used by locking and unlocking
2022 
2023   const Register swap_reg = r0;
2024   const Register obj_reg  = r19;  // Will contain the oop
2025   const Register lock_reg = r13;  // Address of compiler lock object (BasicLock)
2026   const Register old_hdr  = r13;  // value of old header at unlock time
2027   const Register tmp = lr;
2028 
2029   Label slow_path_lock;
2030   Label lock_done;
2031 
2032   if (method->is_synchronized()) {
2033     assert(!is_critical_native, "unhandled");
2034 
2035     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
2036 
2037     // Get the handle (the 2nd argument)
2038     __ mov(oop_handle_reg, c_rarg1);
2039 
2040     // Get address of the box
2041 
2042     __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
2043 
2044     // Load the oop from the handle
2045     __ ldr(obj_reg, Address(oop_handle_reg, 0));
2046 
2047     // Load (object->mark() | 1) into swap_reg %r0
2048     __ ldr(rscratch1, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
2049     __ orr(swap_reg, rscratch1, 1);
2050     if (EnableValhalla) {
2051       // Mask inline_type bit such that we go to the slow path if object is an inline type
2052       __ andr(swap_reg, swap_reg, ~((int) markWord::inline_type_bit_in_place));
2053     }
2054 
2055     // Save (object->mark() | 1) into BasicLock's displaced header
2056     __ str(swap_reg, Address(lock_reg, mark_word_offset));
2057 
2058     // src -> dest iff dest == r0 else r0 <- dest
2059     { Label here;
2060       __ cmpxchg_obj_header(r0, lock_reg, obj_reg, rscratch1, lock_done, /*fallthrough*/NULL);
2061     }
2062 
2063     // Hmm should this move to the slow path code area???
2064 
2065     // Test if the oopMark is an obvious stack pointer, i.e.,
2066     //  1) (mark & 3) == 0, and
2067     //  2) sp <= mark < mark + os::pagesize()
2068     // These 3 tests can be done by evaluating the following
2069     // expression: ((mark - sp) & (3 - os::vm_page_size())),
2070     // assuming both stack pointer and pagesize have their
2071     // least significant 2 bits clear.
2072     // NOTE: the oopMark is in swap_reg %r0 as the result of cmpxchg
2073 
2074     __ sub(swap_reg, sp, swap_reg);
2075     __ neg(swap_reg, swap_reg);
2076     __ ands(swap_reg, swap_reg, 3 - os::vm_page_size());
2077 
2078     // Save the test result, for recursive case, the result is zero
2079     __ str(swap_reg, Address(lock_reg, mark_word_offset));
2080     __ br(Assembler::NE, slow_path_lock);
2081 
2082     // Slow path will re-enter here
2083 
2084     __ bind(lock_done);
2085   }
2086 
2087 
2088   // Finally just about ready to make the JNI call
2089 
2090   // get JNIEnv* which is first argument to native
2091   if (!is_critical_native) {
2092     __ lea(c_rarg0, Address(rthread, in_bytes(JavaThread::jni_environment_offset())));
2093 
2094     // Now set thread in native
2095     __ mov(rscratch1, _thread_in_native);
2096     __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
2097     __ stlrw(rscratch1, rscratch2);
2098   }
2099 
2100   rt_call(masm, native_func);
2101 
2102   __ bind(native_return);
2103 
2104   intptr_t return_pc = (intptr_t) __ pc();
2105   oop_maps->add_gc_map(return_pc - start, map);
2106 
2107   // Unpack native results.
2108   switch (ret_type) {
2109   case T_BOOLEAN: __ c2bool(r0);                     break;
2110   case T_CHAR   : __ ubfx(r0, r0, 0, 16);            break;
2111   case T_BYTE   : __ sbfx(r0, r0, 0, 8);             break;
2112   case T_SHORT  : __ sbfx(r0, r0, 0, 16);            break;
2113   case T_INT    : __ sbfx(r0, r0, 0, 32);            break;
2114   case T_DOUBLE :
2115   case T_FLOAT  :
2116     // Result is in v0 we'll save as needed
2117     break;
2118   case T_ARRAY:                 // Really a handle
2119   case T_INLINE_TYPE:           // Really a handle
2120   case T_OBJECT:                // Really a handle
2121       break; // can't de-handlize until after safepoint check
2122   case T_VOID: break;
2123   case T_LONG: break;
2124   default       : ShouldNotReachHere();
2125   }
2126 
2127   Label safepoint_in_progress, safepoint_in_progress_done;
2128   Label after_transition;
2129 
2130   // If this is a critical native, check for a safepoint or suspend request after the call.
2131   // If a safepoint is needed, transition to native, then to native_trans to handle
2132   // safepoints like the native methods that are not critical natives.
2133   if (is_critical_native) {
2134     Label needs_safepoint;
2135     __ safepoint_poll(needs_safepoint, false /* at_return */, true /* acquire */, false /* in_nmethod */);
2136     __ ldrw(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
2137     __ cbnzw(rscratch1, needs_safepoint);
2138     __ b(after_transition);
2139     __ bind(needs_safepoint);
2140   }
2141 
2142   // Switch thread to "native transition" state before reading the synchronization state.
2143   // This additional state is necessary because reading and testing the synchronization
2144   // state is not atomic w.r.t. GC, as this scenario demonstrates:
2145   //     Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2146   //     VM thread changes sync state to synchronizing and suspends threads for GC.
2147   //     Thread A is resumed to finish this native method, but doesn't block here since it
2148   //     didn't see any synchronization is progress, and escapes.
2149   __ mov(rscratch1, _thread_in_native_trans);
2150 
2151   __ strw(rscratch1, Address(rthread, JavaThread::thread_state_offset()));
2152 
2153   // Force this write out before the read below
2154   __ dmb(Assembler::ISH);
2155 
2156   __ verify_sve_vector_length();
2157 
2158   // Check for safepoint operation in progress and/or pending suspend requests.
2159   {
2160     // We need an acquire here to ensure that any subsequent load of the
2161     // global SafepointSynchronize::_state flag is ordered after this load
2162     // of the thread-local polling word.  We don't want this poll to
2163     // return false (i.e. not safepointing) and a later poll of the global
2164     // SafepointSynchronize::_state spuriously to return true.
2165     //
2166     // This is to avoid a race when we're in a native->Java transition
2167     // racing the code which wakes up from a safepoint.
2168 
2169     __ safepoint_poll(safepoint_in_progress, true /* at_return */, true /* acquire */, false /* in_nmethod */);
2170     __ ldrw(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
2171     __ cbnzw(rscratch1, safepoint_in_progress);
2172     __ bind(safepoint_in_progress_done);
2173   }
2174 
2175   // change thread state
2176   __ mov(rscratch1, _thread_in_Java);
2177   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
2178   __ stlrw(rscratch1, rscratch2);
2179   __ bind(after_transition);
2180 
2181   Label reguard;
2182   Label reguard_done;
2183   __ ldrb(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
2184   __ cmpw(rscratch1, StackOverflow::stack_guard_yellow_reserved_disabled);
2185   __ br(Assembler::EQ, reguard);
2186   __ bind(reguard_done);
2187 
2188   // native result if any is live
2189 
2190   // Unlock
2191   Label unlock_done;
2192   Label slow_path_unlock;
2193   if (method->is_synchronized()) {
2194 
2195     // Get locked oop from the handle we passed to jni
2196     __ ldr(obj_reg, Address(oop_handle_reg, 0));
2197 
2198     Label done;
2199     // Simple recursive lock?
2200 
2201     __ ldr(rscratch1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
2202     __ cbz(rscratch1, done);
2203 
2204     // Must save r0 if if it is live now because cmpxchg must use it
2205     if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2206       save_native_result(masm, ret_type, stack_slots);
2207     }
2208 
2209 
2210     // get address of the stack lock
2211     __ lea(r0, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
2212     //  get old displaced header
2213     __ ldr(old_hdr, Address(r0, 0));
2214 
2215     // Atomic swap old header if oop still contains the stack lock
2216     Label succeed;
2217     __ cmpxchg_obj_header(r0, old_hdr, obj_reg, rscratch1, succeed, &slow_path_unlock);
2218     __ bind(succeed);
2219 
2220     // slow path re-enters here
2221     __ bind(unlock_done);
2222     if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2223       restore_native_result(masm, ret_type, stack_slots);
2224     }
2225 
2226     __ bind(done);
2227   }
2228 
2229   Label dtrace_method_exit, dtrace_method_exit_done;
2230   {
2231     uint64_t offset;
2232     __ adrp(rscratch1, ExternalAddress((address)&DTraceMethodProbes), offset);
2233     __ ldrb(rscratch1, Address(rscratch1, offset));
2234     __ cbnzw(rscratch1, dtrace_method_exit);
2235     __ bind(dtrace_method_exit_done);
2236   }
2237 
2238   __ reset_last_Java_frame(false);
2239 
2240   // Unbox oop result, e.g. JNIHandles::resolve result.
2241   if (is_reference_type(ret_type)) {
2242     __ resolve_jobject(r0, rthread, rscratch2);
2243   }
2244 
2245   if (CheckJNICalls) {
2246     // clear_pending_jni_exception_check
2247     __ str(zr, Address(rthread, JavaThread::pending_jni_exception_check_fn_offset()));
2248   }
2249 
2250   if (!is_critical_native) {
2251     // reset handle block
2252     __ ldr(r2, Address(rthread, JavaThread::active_handles_offset()));
2253     __ str(zr, Address(r2, JNIHandleBlock::top_offset_in_bytes()));
2254   }
2255 
2256   __ leave();
2257 
2258   if (!is_critical_native) {
2259     // Any exception pending?
2260     __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2261     __ cbnz(rscratch1, exception_pending);
2262   }
2263 
2264   // We're done
2265   __ ret(lr);
2266 
2267   // Unexpected paths are out of line and go here
2268 
2269   if (!is_critical_native) {
2270     // forward the exception
2271     __ bind(exception_pending);
2272 
2273     // and forward the exception
2274     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2275   }
2276 
2277   // Slow path locking & unlocking
2278   if (method->is_synchronized()) {
2279 
2280     __ block_comment("Slow path lock {");
2281     __ bind(slow_path_lock);
2282 
2283     // has last_Java_frame setup. No exceptions so do vanilla call not call_VM
2284     // args are (oop obj, BasicLock* lock, JavaThread* thread)
2285 
2286     // protect the args we've loaded
2287     save_args(masm, total_c_args, c_arg, out_regs);
2288 
2289     __ mov(c_rarg0, obj_reg);
2290     __ mov(c_rarg1, lock_reg);
2291     __ mov(c_rarg2, rthread);
2292 
2293     // Not a leaf but we have last_Java_frame setup as we want
2294     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3);
2295     restore_args(masm, total_c_args, c_arg, out_regs);
2296 
2297 #ifdef ASSERT
2298     { Label L;
2299       __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2300       __ cbz(rscratch1, L);
2301       __ stop("no pending exception allowed on exit from monitorenter");
2302       __ bind(L);
2303     }
2304 #endif
2305     __ b(lock_done);
2306 
2307     __ block_comment("} Slow path lock");
2308 
2309     __ block_comment("Slow path unlock {");
2310     __ bind(slow_path_unlock);
2311 
2312     // If we haven't already saved the native result we must save it now as xmm registers
2313     // are still exposed.
2314 
2315     if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
2316       save_native_result(masm, ret_type, stack_slots);
2317     }
2318 
2319     __ mov(c_rarg2, rthread);
2320     __ lea(c_rarg1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
2321     __ mov(c_rarg0, obj_reg);
2322 
2323     // Save pending exception around call to VM (which contains an EXCEPTION_MARK)
2324     // NOTE that obj_reg == r19 currently
2325     __ ldr(r19, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2326     __ str(zr, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2327 
2328     rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C));
2329 
2330 #ifdef ASSERT
2331     {
2332       Label L;
2333       __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2334       __ cbz(rscratch1, L);
2335       __ stop("no pending exception allowed on exit complete_monitor_unlocking_C");
2336       __ bind(L);
2337     }
2338 #endif /* ASSERT */
2339 
2340     __ str(r19, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2341 
2342     if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
2343       restore_native_result(masm, ret_type, stack_slots);
2344     }
2345     __ b(unlock_done);
2346 
2347     __ block_comment("} Slow path unlock");
2348 
2349   } // synchronized
2350 
2351   // SLOW PATH Reguard the stack if needed
2352 
2353   __ bind(reguard);
2354   save_native_result(masm, ret_type, stack_slots);
2355   rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
2356   restore_native_result(masm, ret_type, stack_slots);
2357   // and continue
2358   __ b(reguard_done);
2359 
2360   // SLOW PATH safepoint
2361   {
2362     __ block_comment("safepoint {");
2363     __ bind(safepoint_in_progress);
2364 
2365     // Don't use call_VM as it will see a possible pending exception and forward it
2366     // and never return here preventing us from clearing _last_native_pc down below.
2367     //
2368     save_native_result(masm, ret_type, stack_slots);
2369     __ mov(c_rarg0, rthread);
2370 #ifndef PRODUCT
2371   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
2372 #endif
2373     __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
2374     __ blr(rscratch1);
2375 
2376     // Restore any method result value
2377     restore_native_result(masm, ret_type, stack_slots);
2378 
2379     __ b(safepoint_in_progress_done);
2380     __ block_comment("} safepoint");
2381   }
2382 
2383   // SLOW PATH dtrace support
2384   {
2385     __ block_comment("dtrace entry {");
2386     __ bind(dtrace_method_entry);
2387 
2388     // We have all of the arguments setup at this point. We must not touch any register
2389     // argument registers at this point (what if we save/restore them there are no oop?
2390 
2391     save_args(masm, total_c_args, c_arg, out_regs);
2392     __ mov_metadata(c_rarg1, method());
2393     __ call_VM_leaf(
2394       CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
2395       rthread, c_rarg1);
2396     restore_args(masm, total_c_args, c_arg, out_regs);
2397     __ b(dtrace_method_entry_done);
2398     __ block_comment("} dtrace entry");
2399   }
2400 
2401   {
2402     __ block_comment("dtrace exit {");
2403     __ bind(dtrace_method_exit);
2404     save_native_result(masm, ret_type, stack_slots);
2405     __ mov_metadata(c_rarg1, method());
2406     __ call_VM_leaf(
2407          CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2408          rthread, c_rarg1);
2409     restore_native_result(masm, ret_type, stack_slots);
2410     __ b(dtrace_method_exit_done);
2411     __ block_comment("} dtrace exit");
2412   }
2413 
2414 
2415   __ flush();
2416 
2417   nmethod *nm = nmethod::new_native_nmethod(method,
2418                                             compile_id,
2419                                             masm->code(),
2420                                             vep_offset,
2421                                             frame_complete,
2422                                             stack_slots / VMRegImpl::slots_per_word,
2423                                             (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)),
2424                                             in_ByteSize(lock_slot_offset*VMRegImpl::stack_slot_size),
2425                                             oop_maps);
2426 
2427   return nm;
2428 }
2429 
2430 // this function returns the adjust size (in number of words) to a c2i adapter
2431 // activation for use during deoptimization
2432 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals) {
2433   assert(callee_locals >= callee_parameters,
2434           "test and remove; got more parms than locals");
2435   if (callee_locals < callee_parameters)
2436     return 0;                   // No adjustment for negative locals
2437   int diff = (callee_locals - callee_parameters) * Interpreter::stackElementWords;
2438   // diff is counted in stack words
2439   return align_up(diff, 2);
2440 }
2441 
2442 
2443 //------------------------------generate_deopt_blob----------------------------
2444 void SharedRuntime::generate_deopt_blob() {
2445   // Allocate space for the code
2446   ResourceMark rm;
2447   // Setup code generation tools
2448   int pad = 0;
2449 #if INCLUDE_JVMCI
2450   if (EnableJVMCI) {
2451     pad += 512; // Increase the buffer size when compiling for JVMCI
2452   }
2453 #endif
2454   CodeBuffer buffer("deopt_blob", 2048+pad, 1024);
2455   MacroAssembler* masm = new MacroAssembler(&buffer);
2456   int frame_size_in_words;
2457   OopMap* map = NULL;
2458   OopMapSet *oop_maps = new OopMapSet();
2459   RegisterSaver reg_save(COMPILER2_OR_JVMCI != 0);
2460 
2461   // -------------
2462   // This code enters when returning to a de-optimized nmethod.  A return
2463   // address has been pushed on the the stack, and return values are in
2464   // registers.
2465   // If we are doing a normal deopt then we were called from the patched
2466   // nmethod from the point we returned to the nmethod. So the return
2467   // address on the stack is wrong by NativeCall::instruction_size
2468   // We will adjust the value so it looks like we have the original return
2469   // address on the stack (like when we eagerly deoptimized).
2470   // In the case of an exception pending when deoptimizing, we enter
2471   // with a return address on the stack that points after the call we patched
2472   // into the exception handler. We have the following register state from,
2473   // e.g., the forward exception stub (see stubGenerator_x86_64.cpp).
2474   //    r0: exception oop
2475   //    r19: exception handler
2476   //    r3: throwing pc
2477   // So in this case we simply jam r3 into the useless return address and
2478   // the stack looks just like we want.
2479   //
2480   // At this point we need to de-opt.  We save the argument return
2481   // registers.  We call the first C routine, fetch_unroll_info().  This
2482   // routine captures the return values and returns a structure which
2483   // describes the current frame size and the sizes of all replacement frames.
2484   // The current frame is compiled code and may contain many inlined
2485   // functions, each with their own JVM state.  We pop the current frame, then
2486   // push all the new frames.  Then we call the C routine unpack_frames() to
2487   // populate these frames.  Finally unpack_frames() returns us the new target
2488   // address.  Notice that callee-save registers are BLOWN here; they have
2489   // already been captured in the vframeArray at the time the return PC was
2490   // patched.
2491   address start = __ pc();
2492   Label cont;
2493 
2494   // Prolog for non exception case!
2495 
2496   // Save everything in sight.
2497   map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2498 
2499   // Normal deoptimization.  Save exec mode for unpack_frames.
2500   __ movw(rcpool, Deoptimization::Unpack_deopt); // callee-saved
2501   __ b(cont);
2502 
2503   int reexecute_offset = __ pc() - start;
2504 #if INCLUDE_JVMCI && !defined(COMPILER1)
2505   if (EnableJVMCI && UseJVMCICompiler) {
2506     // JVMCI does not use this kind of deoptimization
2507     __ should_not_reach_here();
2508   }
2509 #endif
2510 
2511   // Reexecute case
2512   // return address is the pc describes what bci to do re-execute at
2513 
2514   // No need to update map as each call to save_live_registers will produce identical oopmap
2515   (void) reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2516 
2517   __ movw(rcpool, Deoptimization::Unpack_reexecute); // callee-saved
2518   __ b(cont);
2519 
2520 #if INCLUDE_JVMCI
2521   Label after_fetch_unroll_info_call;
2522   int implicit_exception_uncommon_trap_offset = 0;
2523   int uncommon_trap_offset = 0;
2524 
2525   if (EnableJVMCI) {
2526     implicit_exception_uncommon_trap_offset = __ pc() - start;
2527 
2528     __ ldr(lr, Address(rthread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())));
2529     __ str(zr, Address(rthread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())));
2530 
2531     uncommon_trap_offset = __ pc() - start;
2532 
2533     // Save everything in sight.
2534     reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2535     // fetch_unroll_info needs to call last_java_frame()
2536     Label retaddr;
2537     __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2538 
2539     __ ldrw(c_rarg1, Address(rthread, in_bytes(JavaThread::pending_deoptimization_offset())));
2540     __ movw(rscratch1, -1);
2541     __ strw(rscratch1, Address(rthread, in_bytes(JavaThread::pending_deoptimization_offset())));
2542 
2543     __ movw(rcpool, (int32_t)Deoptimization::Unpack_reexecute);
2544     __ mov(c_rarg0, rthread);
2545     __ movw(c_rarg2, rcpool); // exec mode
2546     __ lea(rscratch1,
2547            RuntimeAddress(CAST_FROM_FN_PTR(address,
2548                                            Deoptimization::uncommon_trap)));
2549     __ blr(rscratch1);
2550     __ bind(retaddr);
2551     oop_maps->add_gc_map( __ pc()-start, map->deep_copy());
2552 
2553     __ reset_last_Java_frame(false);
2554 
2555     __ b(after_fetch_unroll_info_call);
2556   } // EnableJVMCI
2557 #endif // INCLUDE_JVMCI
2558 
2559   int exception_offset = __ pc() - start;
2560 
2561   // Prolog for exception case
2562 
2563   // all registers are dead at this entry point, except for r0, and
2564   // r3 which contain the exception oop and exception pc
2565   // respectively.  Set them in TLS and fall thru to the
2566   // unpack_with_exception_in_tls entry point.
2567 
2568   __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
2569   __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
2570 
2571   int exception_in_tls_offset = __ pc() - start;
2572 
2573   // new implementation because exception oop is now passed in JavaThread
2574 
2575   // Prolog for exception case
2576   // All registers must be preserved because they might be used by LinearScan
2577   // Exceptiop oop and throwing PC are passed in JavaThread
2578   // tos: stack at point of call to method that threw the exception (i.e. only
2579   // args are on the stack, no return address)
2580 
2581   // The return address pushed by save_live_registers will be patched
2582   // later with the throwing pc. The correct value is not available
2583   // now because loading it from memory would destroy registers.
2584 
2585   // NB: The SP at this point must be the SP of the method that is
2586   // being deoptimized.  Deoptimization assumes that the frame created
2587   // here by save_live_registers is immediately below the method's SP.
2588   // This is a somewhat fragile mechanism.
2589 
2590   // Save everything in sight.
2591   map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
2592 
2593   // Now it is safe to overwrite any register
2594 
2595   // Deopt during an exception.  Save exec mode for unpack_frames.
2596   __ mov(rcpool, Deoptimization::Unpack_exception); // callee-saved
2597 
2598   // load throwing pc from JavaThread and patch it as the return address
2599   // of the current frame. Then clear the field in JavaThread
2600 
2601   __ ldr(r3, Address(rthread, JavaThread::exception_pc_offset()));
2602   __ str(r3, Address(rfp, wordSize));
2603   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
2604 
2605 #ifdef ASSERT
2606   // verify that there is really an exception oop in JavaThread
2607   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
2608   __ verify_oop(r0);
2609 
2610   // verify that there is no pending exception
2611   Label no_pending_exception;
2612   __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
2613   __ cbz(rscratch1, no_pending_exception);
2614   __ stop("must not have pending exception here");
2615   __ bind(no_pending_exception);
2616 #endif
2617 
2618   __ bind(cont);
2619 
2620   // Call C code.  Need thread and this frame, but NOT official VM entry
2621   // crud.  We cannot block on this call, no GC can happen.
2622   //
2623   // UnrollBlock* fetch_unroll_info(JavaThread* thread)
2624 
2625   // fetch_unroll_info needs to call last_java_frame().
2626 
2627   Label retaddr;
2628   __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2629 #ifdef ASSERT0
2630   { Label L;
2631     __ ldr(rscratch1, Address(rthread,
2632                               JavaThread::last_Java_fp_offset()));
2633     __ cbz(rscratch1, L);
2634     __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared");
2635     __ bind(L);
2636   }
2637 #endif // ASSERT
2638   __ mov(c_rarg0, rthread);
2639   __ mov(c_rarg1, rcpool);
2640   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info)));
2641   __ blr(rscratch1);
2642   __ bind(retaddr);
2643 
2644   // Need to have an oopmap that tells fetch_unroll_info where to
2645   // find any register it might need.
2646   oop_maps->add_gc_map(__ pc() - start, map);
2647 
2648   __ reset_last_Java_frame(false);
2649 
2650 #if INCLUDE_JVMCI
2651   if (EnableJVMCI) {
2652     __ bind(after_fetch_unroll_info_call);
2653   }
2654 #endif
2655 
2656   // Load UnrollBlock* into r5
2657   __ mov(r5, r0);
2658 
2659   __ ldrw(rcpool, Address(r5, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes()));
2660    Label noException;
2661   __ cmpw(rcpool, Deoptimization::Unpack_exception);   // Was exception pending?
2662   __ br(Assembler::NE, noException);
2663   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
2664   // QQQ this is useless it was NULL above
2665   __ ldr(r3, Address(rthread, JavaThread::exception_pc_offset()));
2666   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
2667   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
2668 
2669   __ verify_oop(r0);
2670 
2671   // Overwrite the result registers with the exception results.
2672   __ str(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2673   // I think this is useless
2674   // __ str(r3, Address(sp, RegisterSaver::r3_offset_in_bytes()));
2675 
2676   __ bind(noException);
2677 
2678   // Only register save data is on the stack.
2679   // Now restore the result registers.  Everything else is either dead
2680   // or captured in the vframeArray.
2681 
2682   // Restore fp result register
2683   __ ldrd(v0, Address(sp, reg_save.v0_offset_in_bytes()));
2684   // Restore integer result register
2685   __ ldr(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2686 
2687   // Pop all of the register save area off the stack
2688   __ add(sp, sp, frame_size_in_words * wordSize);
2689 
2690   // All of the register save area has been popped of the stack. Only the
2691   // return address remains.
2692 
2693   // Pop all the frames we must move/replace.
2694   //
2695   // Frame picture (youngest to oldest)
2696   // 1: self-frame (no frame link)
2697   // 2: deopting frame  (no frame link)
2698   // 3: caller of deopting frame (could be compiled/interpreted).
2699   //
2700   // Note: by leaving the return address of self-frame on the stack
2701   // and using the size of frame 2 to adjust the stack
2702   // when we are done the return to frame 3 will still be on the stack.
2703 
2704   // Pop deoptimized frame
2705   __ ldrw(r2, Address(r5, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset_in_bytes()));
2706   __ sub(r2, r2, 2 * wordSize);
2707   __ add(sp, sp, r2);
2708   __ ldp(rfp, lr, __ post(sp, 2 * wordSize));
2709   // LR should now be the return address to the caller (3)
2710 
2711 #ifdef ASSERT
2712   // Compilers generate code that bang the stack by as much as the
2713   // interpreter would need. So this stack banging should never
2714   // trigger a fault. Verify that it does not on non product builds.
2715   __ ldrw(r19, Address(r5, Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes()));
2716   __ bang_stack_size(r19, r2);
2717 #endif
2718   // Load address of array of frame pcs into r2
2719   __ ldr(r2, Address(r5, Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
2720 
2721   // Trash the old pc
2722   // __ addptr(sp, wordSize);  FIXME ????
2723 
2724   // Load address of array of frame sizes into r4
2725   __ ldr(r4, Address(r5, Deoptimization::UnrollBlock::frame_sizes_offset_in_bytes()));
2726 
2727   // Load counter into r3
2728   __ ldrw(r3, Address(r5, Deoptimization::UnrollBlock::number_of_frames_offset_in_bytes()));
2729 
2730   // Now adjust the caller's stack to make up for the extra locals
2731   // but record the original sp so that we can save it in the skeletal interpreter
2732   // frame and the stack walking of interpreter_sender will get the unextended sp
2733   // value and not the "real" sp value.
2734 
2735   const Register sender_sp = r6;
2736 
2737   __ mov(sender_sp, sp);
2738   __ ldrw(r19, Address(r5,
2739                        Deoptimization::UnrollBlock::
2740                        caller_adjustment_offset_in_bytes()));
2741   __ sub(sp, sp, r19);
2742 
2743   // Push interpreter frames in a loop
2744   __ mov(rscratch1, (uint64_t)0xDEADDEAD);        // Make a recognizable pattern
2745   __ mov(rscratch2, rscratch1);
2746   Label loop;
2747   __ bind(loop);
2748   __ ldr(r19, Address(__ post(r4, wordSize)));          // Load frame size
2749   __ sub(r19, r19, 2*wordSize);           // We'll push pc and fp by hand
2750   __ ldr(lr, Address(__ post(r2, wordSize)));  // Load pc
2751   __ enter();                           // Save old & set new fp
2752   __ sub(sp, sp, r19);                  // Prolog
2753   // This value is corrected by layout_activation_impl
2754   __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
2755   __ str(sender_sp, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize)); // Make it walkable
2756   __ mov(sender_sp, sp);               // Pass sender_sp to next frame
2757   __ sub(r3, r3, 1);                   // Decrement counter
2758   __ cbnz(r3, loop);
2759 
2760     // Re-push self-frame
2761   __ ldr(lr, Address(r2));
2762   __ enter();
2763 
2764   // Allocate a full sized register save area.  We subtract 2 because
2765   // enter() just pushed 2 words
2766   __ sub(sp, sp, (frame_size_in_words - 2) * wordSize);
2767 
2768   // Restore frame locals after moving the frame
2769   __ strd(v0, Address(sp, reg_save.v0_offset_in_bytes()));
2770   __ str(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2771 
2772   // Call C code.  Need thread but NOT official VM entry
2773   // crud.  We cannot block on this call, no GC can happen.  Call should
2774   // restore return values to their stack-slots with the new SP.
2775   //
2776   // void Deoptimization::unpack_frames(JavaThread* thread, int exec_mode)
2777 
2778   // Use rfp because the frames look interpreted now
2779   // Don't need the precise return PC here, just precise enough to point into this code blob.
2780   address the_pc = __ pc();
2781   __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
2782 
2783   __ mov(c_rarg0, rthread);
2784   __ movw(c_rarg1, rcpool); // second arg: exec_mode
2785   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
2786   __ blr(rscratch1);
2787 
2788   // Set an oopmap for the call site
2789   // Use the same PC we used for the last java frame
2790   oop_maps->add_gc_map(the_pc - start,
2791                        new OopMap( frame_size_in_words, 0 ));
2792 
2793   // Clear fp AND pc
2794   __ reset_last_Java_frame(true);
2795 
2796   // Collect return values
2797   __ ldrd(v0, Address(sp, reg_save.v0_offset_in_bytes()));
2798   __ ldr(r0, Address(sp, reg_save.r0_offset_in_bytes()));
2799   // I think this is useless (throwing pc?)
2800   // __ ldr(r3, Address(sp, RegisterSaver::r3_offset_in_bytes()));
2801 
2802   // Pop self-frame.
2803   __ leave();                           // Epilog
2804 
2805   // Jump to interpreter
2806   __ ret(lr);
2807 
2808   // Make sure all code is generated
2809   masm->flush();
2810 
2811   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
2812   _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
2813 #if INCLUDE_JVMCI
2814   if (EnableJVMCI) {
2815     _deopt_blob->set_uncommon_trap_offset(uncommon_trap_offset);
2816     _deopt_blob->set_implicit_exception_uncommon_trap_offset(implicit_exception_uncommon_trap_offset);
2817   }
2818 #endif
2819 }
2820 
2821 // Number of stack slots between incoming argument block and the start of
2822 // a new frame.  The PROLOG must add this many slots to the stack.  The
2823 // EPILOG must remove this many slots. aarch64 needs two slots for
2824 // return address and fp.
2825 // TODO think this is correct but check
2826 uint SharedRuntime::in_preserve_stack_slots() {
2827   return 4;
2828 }
2829 
2830 uint SharedRuntime::out_preserve_stack_slots() {
2831   return 0;
2832 }
2833 
2834 #ifdef COMPILER2
2835 //------------------------------generate_uncommon_trap_blob--------------------
2836 void SharedRuntime::generate_uncommon_trap_blob() {
2837   // Allocate space for the code
2838   ResourceMark rm;
2839   // Setup code generation tools
2840   CodeBuffer buffer("uncommon_trap_blob", 2048, 1024);
2841   MacroAssembler* masm = new MacroAssembler(&buffer);
2842 
2843   assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
2844 
2845   address start = __ pc();
2846 
2847   // Push self-frame.  We get here with a return address in LR
2848   // and sp should be 16 byte aligned
2849   // push rfp and retaddr by hand
2850   __ stp(rfp, lr, Address(__ pre(sp, -2 * wordSize)));
2851   // we don't expect an arg reg save area
2852 #ifndef PRODUCT
2853   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
2854 #endif
2855   // compiler left unloaded_class_index in j_rarg0 move to where the
2856   // runtime expects it.
2857   if (c_rarg1 != j_rarg0) {
2858     __ movw(c_rarg1, j_rarg0);
2859   }
2860 
2861   // we need to set the past SP to the stack pointer of the stub frame
2862   // and the pc to the address where this runtime call will return
2863   // although actually any pc in this code blob will do).
2864   Label retaddr;
2865   __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2866 
2867   // Call C code.  Need thread but NOT official VM entry
2868   // crud.  We cannot block on this call, no GC can happen.  Call should
2869   // capture callee-saved registers as well as return values.
2870   // Thread is in rdi already.
2871   //
2872   // UnrollBlock* uncommon_trap(JavaThread* thread, jint unloaded_class_index);
2873   //
2874   // n.b. 2 gp args, 0 fp args, integral return type
2875 
2876   __ mov(c_rarg0, rthread);
2877   __ movw(c_rarg2, (unsigned)Deoptimization::Unpack_uncommon_trap);
2878   __ lea(rscratch1,
2879          RuntimeAddress(CAST_FROM_FN_PTR(address,
2880                                          Deoptimization::uncommon_trap)));
2881   __ blr(rscratch1);
2882   __ bind(retaddr);
2883 
2884   // Set an oopmap for the call site
2885   OopMapSet* oop_maps = new OopMapSet();
2886   OopMap* map = new OopMap(SimpleRuntimeFrame::framesize, 0);
2887 
2888   // location of rfp is known implicitly by the frame sender code
2889 
2890   oop_maps->add_gc_map(__ pc() - start, map);
2891 
2892   __ reset_last_Java_frame(false);
2893 
2894   // move UnrollBlock* into r4
2895   __ mov(r4, r0);
2896 
2897 #ifdef ASSERT
2898   { Label L;
2899     __ ldrw(rscratch1, Address(r4, Deoptimization::UnrollBlock::unpack_kind_offset_in_bytes()));
2900     __ cmpw(rscratch1, (unsigned)Deoptimization::Unpack_uncommon_trap);
2901     __ br(Assembler::EQ, L);
2902     __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared");
2903     __ bind(L);
2904   }
2905 #endif
2906 
2907   // Pop all the frames we must move/replace.
2908   //
2909   // Frame picture (youngest to oldest)
2910   // 1: self-frame (no frame link)
2911   // 2: deopting frame  (no frame link)
2912   // 3: caller of deopting frame (could be compiled/interpreted).
2913 
2914   // Pop self-frame.  We have no frame, and must rely only on r0 and sp.
2915   __ add(sp, sp, (SimpleRuntimeFrame::framesize) << LogBytesPerInt); // Epilog!
2916 
2917   // Pop deoptimized frame (int)
2918   __ ldrw(r2, Address(r4,
2919                       Deoptimization::UnrollBlock::
2920                       size_of_deoptimized_frame_offset_in_bytes()));
2921   __ sub(r2, r2, 2 * wordSize);
2922   __ add(sp, sp, r2);
2923   __ ldp(rfp, lr, __ post(sp, 2 * wordSize));
2924   // LR should now be the return address to the caller (3) frame
2925 
2926 #ifdef ASSERT
2927   // Compilers generate code that bang the stack by as much as the
2928   // interpreter would need. So this stack banging should never
2929   // trigger a fault. Verify that it does not on non product builds.
2930   __ ldrw(r1, Address(r4,
2931                       Deoptimization::UnrollBlock::
2932                       total_frame_sizes_offset_in_bytes()));
2933   __ bang_stack_size(r1, r2);
2934 #endif
2935 
2936   // Load address of array of frame pcs into r2 (address*)
2937   __ ldr(r2, Address(r4,
2938                      Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
2939 
2940   // Load address of array of frame sizes into r5 (intptr_t*)
2941   __ ldr(r5, Address(r4,
2942                      Deoptimization::UnrollBlock::
2943                      frame_sizes_offset_in_bytes()));
2944 
2945   // Counter
2946   __ ldrw(r3, Address(r4,
2947                       Deoptimization::UnrollBlock::
2948                       number_of_frames_offset_in_bytes())); // (int)
2949 
2950   // Now adjust the caller's stack to make up for the extra locals but
2951   // record the original sp so that we can save it in the skeletal
2952   // interpreter frame and the stack walking of interpreter_sender
2953   // will get the unextended sp value and not the "real" sp value.
2954 
2955   const Register sender_sp = r8;
2956 
2957   __ mov(sender_sp, sp);
2958   __ ldrw(r1, Address(r4,
2959                       Deoptimization::UnrollBlock::
2960                       caller_adjustment_offset_in_bytes())); // (int)
2961   __ sub(sp, sp, r1);
2962 
2963   // Push interpreter frames in a loop
2964   Label loop;
2965   __ bind(loop);
2966   __ ldr(r1, Address(r5, 0));       // Load frame size
2967   __ sub(r1, r1, 2 * wordSize);     // We'll push pc and rfp by hand
2968   __ ldr(lr, Address(r2, 0));       // Save return address
2969   __ enter();                       // and old rfp & set new rfp
2970   __ sub(sp, sp, r1);               // Prolog
2971   __ str(sender_sp, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize)); // Make it walkable
2972   // This value is corrected by layout_activation_impl
2973   __ str(zr, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
2974   __ mov(sender_sp, sp);          // Pass sender_sp to next frame
2975   __ add(r5, r5, wordSize);       // Bump array pointer (sizes)
2976   __ add(r2, r2, wordSize);       // Bump array pointer (pcs)
2977   __ subsw(r3, r3, 1);            // Decrement counter
2978   __ br(Assembler::GT, loop);
2979   __ ldr(lr, Address(r2, 0));     // save final return address
2980   // Re-push self-frame
2981   __ enter();                     // & old rfp & set new rfp
2982 
2983   // Use rfp because the frames look interpreted now
2984   // Save "the_pc" since it cannot easily be retrieved using the last_java_SP after we aligned SP.
2985   // Don't need the precise return PC here, just precise enough to point into this code blob.
2986   address the_pc = __ pc();
2987   __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
2988 
2989   // Call C code.  Need thread but NOT official VM entry
2990   // crud.  We cannot block on this call, no GC can happen.  Call should
2991   // restore return values to their stack-slots with the new SP.
2992   // Thread is in rdi already.
2993   //
2994   // BasicType unpack_frames(JavaThread* thread, int exec_mode);
2995   //
2996   // n.b. 2 gp args, 0 fp args, integral return type
2997 
2998   // sp should already be aligned
2999   __ mov(c_rarg0, rthread);
3000   __ movw(c_rarg1, (unsigned)Deoptimization::Unpack_uncommon_trap);
3001   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
3002   __ blr(rscratch1);
3003 
3004   // Set an oopmap for the call site
3005   // Use the same PC we used for the last java frame
3006   oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0));
3007 
3008   // Clear fp AND pc
3009   __ reset_last_Java_frame(true);
3010 
3011   // Pop self-frame.
3012   __ leave();                 // Epilog
3013 
3014   // Jump to interpreter
3015   __ ret(lr);
3016 
3017   // Make sure all code is generated
3018   masm->flush();
3019 
3020   _uncommon_trap_blob =  UncommonTrapBlob::create(&buffer, oop_maps,
3021                                                  SimpleRuntimeFrame::framesize >> 1);
3022 }
3023 #endif // COMPILER2
3024 
3025 
3026 //------------------------------generate_handler_blob------
3027 //
3028 // Generate a special Compile2Runtime blob that saves all registers,
3029 // and setup oopmap.
3030 //
3031 SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_type) {
3032   ResourceMark rm;
3033   OopMapSet *oop_maps = new OopMapSet();
3034   OopMap* map;
3035 
3036   // Allocate space for the code.  Setup code generation tools.
3037   CodeBuffer buffer("handler_blob", 2048, 1024);
3038   MacroAssembler* masm = new MacroAssembler(&buffer);
3039 
3040   address start   = __ pc();
3041   address call_pc = NULL;
3042   int frame_size_in_words;
3043   bool cause_return = (poll_type == POLL_AT_RETURN);
3044   RegisterSaver reg_save(poll_type == POLL_AT_VECTOR_LOOP /* save_vectors */);
3045 
3046   // Save Integer and Float registers.
3047   map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
3048 
3049   // The following is basically a call_VM.  However, we need the precise
3050   // address of the call in order to generate an oopmap. Hence, we do all the
3051   // work outselves.
3052 
3053   Label retaddr;
3054   __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
3055 
3056   // The return address must always be correct so that frame constructor never
3057   // sees an invalid pc.
3058 
3059   if (!cause_return) {
3060     // overwrite the return address pushed by save_live_registers
3061     // Additionally, r20 is a callee-saved register so we can look at
3062     // it later to determine if someone changed the return address for
3063     // us!
3064     __ ldr(r20, Address(rthread, JavaThread::saved_exception_pc_offset()));
3065     __ str(r20, Address(rfp, wordSize));
3066   }
3067 
3068   // Do the call
3069   __ mov(c_rarg0, rthread);
3070   __ lea(rscratch1, RuntimeAddress(call_ptr));
3071   __ blr(rscratch1);
3072   __ bind(retaddr);
3073 
3074   // Set an oopmap for the call site.  This oopmap will map all
3075   // oop-registers and debug-info registers as callee-saved.  This
3076   // will allow deoptimization at this safepoint to find all possible
3077   // debug-info recordings, as well as let GC find all oops.
3078 
3079   oop_maps->add_gc_map( __ pc() - start, map);
3080 
3081   Label noException;
3082 
3083   __ reset_last_Java_frame(false);
3084 
3085   __ membar(Assembler::LoadLoad | Assembler::LoadStore);
3086 
3087   __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
3088   __ cbz(rscratch1, noException);
3089 
3090   // Exception pending
3091 
3092   reg_save.restore_live_registers(masm);
3093 
3094   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3095 
3096   // No exception case
3097   __ bind(noException);
3098 
3099   Label no_adjust, bail;
3100   if (!cause_return) {
3101     // If our stashed return pc was modified by the runtime we avoid touching it
3102     __ ldr(rscratch1, Address(rfp, wordSize));
3103     __ cmp(r20, rscratch1);
3104     __ br(Assembler::NE, no_adjust);
3105 
3106 #ifdef ASSERT
3107     // Verify the correct encoding of the poll we're about to skip.
3108     // See NativeInstruction::is_ldrw_to_zr()
3109     __ ldrw(rscratch1, Address(r20));
3110     __ ubfx(rscratch2, rscratch1, 22, 10);
3111     __ cmpw(rscratch2, 0b1011100101);
3112     __ br(Assembler::NE, bail);
3113     __ ubfx(rscratch2, rscratch1, 0, 5);
3114     __ cmpw(rscratch2, 0b11111);
3115     __ br(Assembler::NE, bail);
3116 #endif
3117     // Adjust return pc forward to step over the safepoint poll instruction
3118     __ add(r20, r20, NativeInstruction::instruction_size);
3119     __ str(r20, Address(rfp, wordSize));
3120   }
3121 
3122   __ bind(no_adjust);
3123   // Normal exit, restore registers and exit.
3124   reg_save.restore_live_registers(masm);
3125 
3126   __ ret(lr);
3127 
3128 #ifdef ASSERT
3129   __ bind(bail);
3130   __ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected");
3131 #endif
3132 
3133   // Make sure all code is generated
3134   masm->flush();
3135 
3136   // Fill-out other meta info
3137   return SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
3138 }
3139 
3140 //
3141 // generate_resolve_blob - call resolution (static/virtual/opt-virtual/ic-miss
3142 //
3143 // Generate a stub that calls into vm to find out the proper destination
3144 // of a java call. All the argument registers are live at this point
3145 // but since this is generic code we don't know what they are and the caller
3146 // must do any gc of the args.
3147 //
3148 RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const char* name) {
3149   assert (StubRoutines::forward_exception_entry() != NULL, "must be generated before");
3150 
3151   // allocate space for the code
3152   ResourceMark rm;
3153 
3154   CodeBuffer buffer(name, 1000, 512);
3155   MacroAssembler* masm                = new MacroAssembler(&buffer);
3156 
3157   int frame_size_in_words;
3158   RegisterSaver reg_save(false /* save_vectors */);
3159 
3160   OopMapSet *oop_maps = new OopMapSet();
3161   OopMap* map = NULL;
3162 
3163   int start = __ offset();
3164 
3165   map = reg_save.save_live_registers(masm, 0, &frame_size_in_words);
3166 
3167   int frame_complete = __ offset();
3168 
3169   {
3170     Label retaddr;
3171     __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
3172 
3173     __ mov(c_rarg0, rthread);
3174     __ lea(rscratch1, RuntimeAddress(destination));
3175 
3176     __ blr(rscratch1);
3177     __ bind(retaddr);
3178   }
3179 
3180   // Set an oopmap for the call site.
3181   // We need this not only for callee-saved registers, but also for volatile
3182   // registers that the compiler might be keeping live across a safepoint.
3183 
3184   oop_maps->add_gc_map( __ offset() - start, map);
3185 
3186   // r0 contains the address we are going to jump to assuming no exception got installed
3187 
3188   // clear last_Java_sp
3189   __ reset_last_Java_frame(false);
3190   // check for pending exceptions
3191   Label pending;
3192   __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
3193   __ cbnz(rscratch1, pending);
3194 
3195   // get the returned Method*
3196   __ get_vm_result_2(rmethod, rthread);
3197   __ str(rmethod, Address(sp, reg_save.reg_offset_in_bytes(rmethod)));
3198 
3199   // r0 is where we want to jump, overwrite rscratch1 which is saved and scratch
3200   __ str(r0, Address(sp, reg_save.rscratch1_offset_in_bytes()));
3201   reg_save.restore_live_registers(masm);
3202 
3203   // We are back the the original state on entry and ready to go.
3204 
3205   __ br(rscratch1);
3206 
3207   // Pending exception after the safepoint
3208 
3209   __ bind(pending);
3210 
3211   reg_save.restore_live_registers(masm);
3212 
3213   // exception pending => remove activation and forward to exception handler
3214 
3215   __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
3216 
3217   __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
3218   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3219 
3220   // -------------
3221   // make sure all code is generated
3222   masm->flush();
3223 
3224   // return the  blob
3225   // frame_size_words or bytes??
3226   return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true);
3227 }
3228 
3229 #ifdef COMPILER2
3230 // This is here instead of runtime_x86_64.cpp because it uses SimpleRuntimeFrame
3231 //
3232 //------------------------------generate_exception_blob---------------------------
3233 // creates exception blob at the end
3234 // Using exception blob, this code is jumped from a compiled method.
3235 // (see emit_exception_handler in x86_64.ad file)
3236 //
3237 // Given an exception pc at a call we call into the runtime for the
3238 // handler in this method. This handler might merely restore state
3239 // (i.e. callee save registers) unwind the frame and jump to the
3240 // exception handler for the nmethod if there is no Java level handler
3241 // for the nmethod.
3242 //
3243 // This code is entered with a jmp.
3244 //
3245 // Arguments:
3246 //   r0: exception oop
3247 //   r3: exception pc
3248 //
3249 // Results:
3250 //   r0: exception oop
3251 //   r3: exception pc in caller or ???
3252 //   destination: exception handler of caller
3253 //
3254 // Note: the exception pc MUST be at a call (precise debug information)
3255 //       Registers r0, r3, r2, r4, r5, r8-r11 are not callee saved.
3256 //
3257 
3258 void OptoRuntime::generate_exception_blob() {
3259   assert(!OptoRuntime::is_callee_saved_register(R3_num), "");
3260   assert(!OptoRuntime::is_callee_saved_register(R0_num), "");
3261   assert(!OptoRuntime::is_callee_saved_register(R2_num), "");
3262 
3263   assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
3264 
3265   // Allocate space for the code
3266   ResourceMark rm;
3267   // Setup code generation tools
3268   CodeBuffer buffer("exception_blob", 2048, 1024);
3269   MacroAssembler* masm = new MacroAssembler(&buffer);
3270 
3271   // TODO check various assumptions made here
3272   //
3273   // make sure we do so before running this
3274 
3275   address start = __ pc();
3276 
3277   // push rfp and retaddr by hand
3278   // Exception pc is 'return address' for stack walker
3279   __ stp(rfp, lr, Address(__ pre(sp, -2 * wordSize)));
3280   // there are no callee save registers and we don't expect an
3281   // arg reg save area
3282 #ifndef PRODUCT
3283   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
3284 #endif
3285   // Store exception in Thread object. We cannot pass any arguments to the
3286   // handle_exception call, since we do not want to make any assumption
3287   // about the size of the frame where the exception happened in.
3288   __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
3289   __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
3290 
3291   // This call does all the hard work.  It checks if an exception handler
3292   // exists in the method.
3293   // If so, it returns the handler address.
3294   // If not, it prepares for stack-unwinding, restoring the callee-save
3295   // registers of the frame being removed.
3296   //
3297   // address OptoRuntime::handle_exception_C(JavaThread* thread)
3298   //
3299   // n.b. 1 gp arg, 0 fp args, integral return type
3300 
3301   // the stack should always be aligned
3302   address the_pc = __ pc();
3303   __ set_last_Java_frame(sp, noreg, the_pc, rscratch1);
3304   __ mov(c_rarg0, rthread);
3305   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, OptoRuntime::handle_exception_C)));
3306   __ blr(rscratch1);
3307   // handle_exception_C is a special VM call which does not require an explicit
3308   // instruction sync afterwards.
3309 
3310   // May jump to SVE compiled code
3311   __ reinitialize_ptrue();
3312 
3313   // Set an oopmap for the call site.  This oopmap will only be used if we
3314   // are unwinding the stack.  Hence, all locations will be dead.
3315   // Callee-saved registers will be the same as the frame above (i.e.,
3316   // handle_exception_stub), since they were restored when we got the
3317   // exception.
3318 
3319   OopMapSet* oop_maps = new OopMapSet();
3320 
3321   oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0));
3322 
3323   __ reset_last_Java_frame(false);
3324 
3325   // Restore callee-saved registers
3326 
3327   // rfp is an implicitly saved callee saved register (i.e. the calling
3328   // convention will save restore it in prolog/epilog) Other than that
3329   // there are no callee save registers now that adapter frames are gone.
3330   // and we dont' expect an arg reg save area
3331   __ ldp(rfp, r3, Address(__ post(sp, 2 * wordSize)));
3332 
3333   // r0: exception handler
3334 
3335   // We have a handler in r0 (could be deopt blob).
3336   __ mov(r8, r0);
3337 
3338   // Get the exception oop
3339   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
3340   // Get the exception pc in case we are deoptimized
3341   __ ldr(r4, Address(rthread, JavaThread::exception_pc_offset()));
3342 #ifdef ASSERT
3343   __ str(zr, Address(rthread, JavaThread::exception_handler_pc_offset()));
3344   __ str(zr, Address(rthread, JavaThread::exception_pc_offset()));
3345 #endif
3346   // Clear the exception oop so GC no longer processes it as a root.
3347   __ str(zr, Address(rthread, JavaThread::exception_oop_offset()));
3348 
3349   // r0: exception oop
3350   // r8:  exception handler
3351   // r4: exception pc
3352   // Jump to handler
3353 
3354   __ br(r8);
3355 
3356   // Make sure all code is generated
3357   masm->flush();
3358 
3359   // Set exception blob
3360   _exception_blob =  ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
3361 }
3362 #endif // COMPILER2
3363 
3364 BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) {
3365   BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K);
3366   CodeBuffer buffer(buf);
3367   short buffer_locs[20];
3368   buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
3369                                          sizeof(buffer_locs)/sizeof(relocInfo));
3370 
3371   MacroAssembler _masm(&buffer);
3372   MacroAssembler* masm = &_masm;
3373 
3374   const Array<SigEntry>* sig_vk = vk->extended_sig();
3375   const Array<VMRegPair>* regs = vk->return_regs();
3376 
3377   int pack_fields_jobject_off = __ offset();
3378   // Resolve pre-allocated buffer from JNI handle.
3379   // We cannot do this in generate_call_stub() because it requires GC code to be initialized.
3380   Register Rresult = r14;  // See StubGenerator::generate_call_stub().
3381   __ ldr(r0, Address(Rresult));
3382   __ resolve_jobject(r0 /* value */,
3383                      rthread /* thread */,
3384                      r12 /* tmp */);
3385   __ str(r0, Address(Rresult));
3386 
3387   int pack_fields_off = __ offset();
3388 
3389   int j = 1;
3390   for (int i = 0; i < sig_vk->length(); i++) {
3391     BasicType bt = sig_vk->at(i)._bt;
3392     if (bt == T_INLINE_TYPE) {
3393       continue;
3394     }
3395     if (bt == T_VOID) {
3396       if (sig_vk->at(i-1)._bt == T_LONG ||
3397           sig_vk->at(i-1)._bt == T_DOUBLE) {
3398         j++;
3399       }
3400       continue;
3401     }
3402     int off = sig_vk->at(i)._offset;
3403     VMRegPair pair = regs->at(j);
3404     VMReg r_1 = pair.first();
3405     VMReg r_2 = pair.second();
3406     Address to(r0, off);
3407     if (bt == T_FLOAT) {
3408       __ strs(r_1->as_FloatRegister(), to);
3409     } else if (bt == T_DOUBLE) {
3410       __ strd(r_1->as_FloatRegister(), to);
3411     } else if (bt == T_OBJECT || bt == T_ARRAY) {
3412       Register val = r_1->as_Register();
3413       assert_different_registers(r0, val);
3414       // We don't need barriers because the destination is a newly allocated object.
3415       // Also, we cannot use store_heap_oop(to, val) because it uses r8 as tmp.
3416       if (UseCompressedOops) {
3417         __ encode_heap_oop(val);
3418         __ str(val, to);
3419       } else {
3420         __ str(val, to);
3421       }
3422     } else {
3423       assert(is_java_primitive(bt), "unexpected basic type");
3424       assert_different_registers(r0, r_1->as_Register());
3425       size_t size_in_bytes = type2aelembytes(bt);
3426       __ store_sized_value(to, r_1->as_Register(), size_in_bytes);
3427     }
3428     j++;
3429   }
3430   assert(j == regs->length(), "missed a field?");
3431 
3432   __ ret(lr);
3433 
3434   int unpack_fields_off = __ offset();
3435 
3436   j = 1;
3437   for (int i = 0; i < sig_vk->length(); i++) {
3438     BasicType bt = sig_vk->at(i)._bt;
3439     if (bt == T_INLINE_TYPE) {
3440       continue;
3441     }
3442     if (bt == T_VOID) {
3443       if (sig_vk->at(i-1)._bt == T_LONG ||
3444           sig_vk->at(i-1)._bt == T_DOUBLE) {
3445         j++;
3446       }
3447       continue;
3448     }
3449     int off = sig_vk->at(i)._offset;
3450     assert(off > 0, "offset in object should be positive");
3451     VMRegPair pair = regs->at(j);
3452     VMReg r_1 = pair.first();
3453     VMReg r_2 = pair.second();
3454     Address from(r0, off);
3455     if (bt == T_FLOAT) {
3456       __ ldrs(r_1->as_FloatRegister(), from);
3457     } else if (bt == T_DOUBLE) {
3458       __ ldrd(r_1->as_FloatRegister(), from);
3459     } else if (bt == T_OBJECT || bt == T_ARRAY) {
3460       assert_different_registers(r0, r_1->as_Register());
3461       __ load_heap_oop(r_1->as_Register(), from);
3462     } else {
3463       assert(is_java_primitive(bt), "unexpected basic type");
3464       assert_different_registers(r0, r_1->as_Register());
3465 
3466       size_t size_in_bytes = type2aelembytes(bt);
3467       __ load_sized_value(r_1->as_Register(), from, size_in_bytes, bt != T_CHAR && bt != T_BOOLEAN);
3468     }
3469     j++;
3470   }
3471   assert(j == regs->length(), "missed a field?");
3472 
3473   if (StressInlineTypeReturnedAsFields) {
3474     __ load_klass(r0, r0);
3475     __ orr(r0, r0, 1);
3476   }
3477 
3478   __ ret(lr);
3479 
3480   __ flush();
3481 
3482   return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
3483 }
3484 
3485 // ---------------------------------------------------------------
3486 
3487 class NativeInvokerGenerator : public StubCodeGenerator {
3488   address _call_target;
3489   int _shadow_space_bytes;
3490 
3491   const GrowableArray<VMReg>& _input_registers;
3492   const GrowableArray<VMReg>& _output_registers;
3493 
3494   int _frame_complete;
3495   int _framesize;
3496   OopMapSet* _oop_maps;
3497 public:
3498   NativeInvokerGenerator(CodeBuffer* buffer,
3499                          address call_target,
3500                          int shadow_space_bytes,
3501                          const GrowableArray<VMReg>& input_registers,
3502                          const GrowableArray<VMReg>& output_registers)
3503    : StubCodeGenerator(buffer, PrintMethodHandleStubs),
3504      _call_target(call_target),
3505      _shadow_space_bytes(shadow_space_bytes),
3506      _input_registers(input_registers),
3507      _output_registers(output_registers),
3508      _frame_complete(0),
3509      _framesize(0),
3510      _oop_maps(NULL) {
3511     assert(_output_registers.length() <= 1
3512            || (_output_registers.length() == 2 && !_output_registers.at(1)->is_valid()), "no multi-reg returns");
3513   }
3514 
3515   void generate();
3516 
3517   int spill_size_in_bytes() const {
3518     if (_output_registers.length() == 0) {
3519       return 0;
3520     }
3521     VMReg reg = _output_registers.at(0);
3522     assert(reg->is_reg(), "must be a register");
3523     if (reg->is_Register()) {
3524       return 8;
3525     } else if (reg->is_FloatRegister()) {
3526       bool use_sve = Matcher::supports_scalable_vector();
3527       if (use_sve) {
3528         return Matcher::scalable_vector_reg_size(T_BYTE);
3529       }
3530       return 16;
3531     } else {
3532       ShouldNotReachHere();
3533     }
3534     return 0;
3535   }
3536 
3537   void spill_output_registers() {
3538     if (_output_registers.length() == 0) {
3539       return;
3540     }
3541     VMReg reg = _output_registers.at(0);
3542     assert(reg->is_reg(), "must be a register");
3543     MacroAssembler* masm = _masm;
3544     if (reg->is_Register()) {
3545       __ spill(reg->as_Register(), true, 0);
3546     } else if (reg->is_FloatRegister()) {
3547       bool use_sve = Matcher::supports_scalable_vector();
3548       if (use_sve) {
3549         __ spill_sve_vector(reg->as_FloatRegister(), 0, Matcher::scalable_vector_reg_size(T_BYTE));
3550       } else {
3551         __ spill(reg->as_FloatRegister(), __ Q, 0);
3552       }
3553     } else {
3554       ShouldNotReachHere();
3555     }
3556   }
3557 
3558   void fill_output_registers() {
3559     if (_output_registers.length() == 0) {
3560       return;
3561     }
3562     VMReg reg = _output_registers.at(0);
3563     assert(reg->is_reg(), "must be a register");
3564     MacroAssembler* masm = _masm;
3565     if (reg->is_Register()) {
3566       __ unspill(reg->as_Register(), true, 0);
3567     } else if (reg->is_FloatRegister()) {
3568       bool use_sve = Matcher::supports_scalable_vector();
3569       if (use_sve) {
3570         __ unspill_sve_vector(reg->as_FloatRegister(), 0, Matcher::scalable_vector_reg_size(T_BYTE));
3571       } else {
3572         __ unspill(reg->as_FloatRegister(), __ Q, 0);
3573       }
3574     } else {
3575       ShouldNotReachHere();
3576     }
3577   }
3578 
3579   int frame_complete() const {
3580     return _frame_complete;
3581   }
3582 
3583   int framesize() const {
3584     return (_framesize >> (LogBytesPerWord - LogBytesPerInt));
3585   }
3586 
3587   OopMapSet* oop_maps() const {
3588     return _oop_maps;
3589   }
3590 
3591 private:
3592 #ifdef ASSERT
3593   bool target_uses_register(VMReg reg) {
3594     return _input_registers.contains(reg) || _output_registers.contains(reg);
3595   }
3596 #endif
3597 };
3598 
3599 static const int native_invoker_code_size = 1024;
3600 
3601 RuntimeStub* SharedRuntime::make_native_invoker(address call_target,
3602                                                 int shadow_space_bytes,
3603                                                 const GrowableArray<VMReg>& input_registers,
3604                                                 const GrowableArray<VMReg>& output_registers) {
3605   int locs_size  = 64;
3606   CodeBuffer code("nep_invoker_blob", native_invoker_code_size, locs_size);
3607   NativeInvokerGenerator g(&code, call_target, shadow_space_bytes, input_registers, output_registers);
3608   g.generate();
3609   code.log_section_sizes("nep_invoker_blob");
3610 
3611   RuntimeStub* stub =
3612     RuntimeStub::new_runtime_stub("nep_invoker_blob",
3613                                   &code,
3614                                   g.frame_complete(),
3615                                   g.framesize(),
3616                                   g.oop_maps(), false);
3617   return stub;
3618 }
3619 
3620 void NativeInvokerGenerator::generate() {
3621   assert(!(target_uses_register(rscratch1->as_VMReg())
3622            || target_uses_register(rscratch2->as_VMReg())
3623            || target_uses_register(rthread->as_VMReg())),
3624          "Register conflict");
3625 
3626   enum layout {
3627     rbp_off,
3628     rbp_off2,
3629     return_off,
3630     return_off2,
3631     framesize // inclusive of return address
3632   };
3633 
3634   assert(_shadow_space_bytes == 0, "not expecting shadow space on AArch64");
3635   _framesize = align_up(framesize + (spill_size_in_bytes() >> LogBytesPerInt), 4);
3636   assert(is_even(_framesize/2), "sp not 16-byte aligned");
3637 
3638   _oop_maps  = new OopMapSet();
3639   MacroAssembler* masm = _masm;
3640 
3641   address start = __ pc();
3642 
3643   __ enter();
3644 
3645   // lr and fp are already in place
3646   __ sub(sp, rfp, ((unsigned)_framesize-4) << LogBytesPerInt); // prolog
3647 
3648   _frame_complete = __ pc() - start;
3649 
3650   address the_pc = __ pc();
3651   __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
3652   OopMap* map = new OopMap(_framesize, 0);
3653   _oop_maps->add_gc_map(the_pc - start, map);
3654 
3655   // State transition
3656   __ mov(rscratch1, _thread_in_native);
3657   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
3658   __ stlrw(rscratch1, rscratch2);
3659 
3660   rt_call(masm, _call_target);
3661 
3662   __ mov(rscratch1, _thread_in_native_trans);
3663   __ strw(rscratch1, Address(rthread, JavaThread::thread_state_offset()));
3664 
3665   // Force this write out before the read below
3666   __ membar(Assembler::LoadLoad | Assembler::LoadStore |
3667             Assembler::StoreLoad | Assembler::StoreStore);
3668 
3669   __ verify_sve_vector_length();
3670 
3671   Label L_after_safepoint_poll;
3672   Label L_safepoint_poll_slow_path;
3673 
3674   __ safepoint_poll(L_safepoint_poll_slow_path, true /* at_return */, true /* acquire */, false /* in_nmethod */);
3675 
3676   __ ldrw(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
3677   __ cbnzw(rscratch1, L_safepoint_poll_slow_path);
3678 
3679   __ bind(L_after_safepoint_poll);
3680 
3681   // change thread state
3682   __ mov(rscratch1, _thread_in_Java);
3683   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
3684   __ stlrw(rscratch1, rscratch2);
3685 
3686   __ block_comment("reguard stack check");
3687   Label L_reguard;
3688   Label L_after_reguard;
3689   __ ldrb(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
3690   __ cmpw(rscratch1, StackOverflow::stack_guard_yellow_reserved_disabled);
3691   __ br(Assembler::EQ, L_reguard);
3692   __ bind(L_after_reguard);
3693 
3694   __ reset_last_Java_frame(true);
3695 
3696   __ leave(); // required for proper stackwalking of RuntimeStub frame
3697   __ ret(lr);
3698 
3699   //////////////////////////////////////////////////////////////////////////////
3700 
3701   __ block_comment("{ L_safepoint_poll_slow_path");
3702   __ bind(L_safepoint_poll_slow_path);
3703 
3704   // Need to save the native result registers around any runtime calls.
3705   spill_output_registers();
3706 
3707   __ mov(c_rarg0, rthread);
3708   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
3709   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
3710   __ blr(rscratch1);
3711 
3712   fill_output_registers();
3713 
3714   __ b(L_after_safepoint_poll);
3715   __ block_comment("} L_safepoint_poll_slow_path");
3716 
3717   //////////////////////////////////////////////////////////////////////////////
3718 
3719   __ block_comment("{ L_reguard");
3720   __ bind(L_reguard);
3721 
3722   spill_output_registers();
3723 
3724   rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
3725 
3726   fill_output_registers();
3727 
3728   __ b(L_after_reguard);
3729 
3730   __ block_comment("} L_reguard");
3731 
3732   //////////////////////////////////////////////////////////////////////////////
3733 
3734   __ flush();
3735 }