1 /*
   2  * Copyright (c) 2003, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #ifndef _WINDOWS
  26 #include "alloca.h"
  27 #endif
  28 #include "asm/macroAssembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"
  30 #include "classfile/symbolTable.hpp"
  31 #include "code/aotCodeCache.hpp"
  32 #include "code/compiledIC.hpp"
  33 #include "code/debugInfoRec.hpp"
  34 #include "code/nativeInst.hpp"
  35 #include "code/vtableStubs.hpp"
  36 #include "compiler/oopMap.hpp"
  37 #include "gc/shared/collectedHeap.hpp"
  38 #include "gc/shared/gcLocker.hpp"
  39 #include "gc/shared/barrierSet.hpp"
  40 #include "gc/shared/barrierSetAssembler.hpp"
  41 #include "interpreter/interpreter.hpp"
  42 #include "logging/log.hpp"
  43 #include "memory/resourceArea.hpp"
  44 #include "memory/universe.hpp"
  45 #include "oops/klass.inline.hpp"
  46 #include "oops/method.inline.hpp"
  47 #include "prims/methodHandles.hpp"
  48 #include "runtime/continuation.hpp"
  49 #include "runtime/continuationEntry.inline.hpp"
  50 #include "runtime/globals.hpp"
  51 #include "runtime/jniHandles.hpp"
  52 #include "runtime/safepointMechanism.hpp"
  53 #include "runtime/sharedRuntime.hpp"
  54 #include "runtime/signature.hpp"
  55 #include "runtime/stubRoutines.hpp"
  56 #include "runtime/timerTrace.hpp"
  57 #include "runtime/vframeArray.hpp"
  58 #include "runtime/vm_version.hpp"
  59 #include "utilities/align.hpp"
  60 #include "utilities/checkedCast.hpp"
  61 #include "utilities/formatBuffer.hpp"
  62 #include "vmreg_x86.inline.hpp"
  63 #ifdef COMPILER1
  64 #include "c1/c1_Runtime1.hpp"
  65 #endif
  66 #ifdef COMPILER2
  67 #include "opto/runtime.hpp"
  68 #endif
  69 #if INCLUDE_JVMCI
  70 #include "jvmci/jvmciJavaClasses.hpp"
  71 #endif
  72 
  73 #define __ masm->
  74 
  75 #ifdef PRODUCT
  76 #define BLOCK_COMMENT(str) /* nothing */
  77 #else
  78 #define BLOCK_COMMENT(str) __ block_comment(str)
  79 #endif // PRODUCT
  80 
  81 const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;
  82 
  83 class RegisterSaver {
  84   // Capture info about frame layout.  Layout offsets are in jint
  85   // units because compiler frame slots are jints.
  86 #define XSAVE_AREA_BEGIN 160
  87 #define XSAVE_AREA_YMM_BEGIN 576
  88 #define XSAVE_AREA_EGPRS 960
  89 #define XSAVE_AREA_OPMASK_BEGIN 1088
  90 #define XSAVE_AREA_ZMM_BEGIN 1152
  91 #define XSAVE_AREA_UPPERBANK 1664
  92 #define DEF_XMM_OFFS(regnum)       xmm ## regnum ## _off = xmm_off + (regnum)*16/BytesPerInt, xmm ## regnum ## H_off
  93 #define DEF_YMM_OFFS(regnum)       ymm ## regnum ## _off = ymm_off + (regnum)*16/BytesPerInt, ymm ## regnum ## H_off
  94 #define DEF_ZMM_OFFS(regnum)       zmm ## regnum ## _off = zmm_off + (regnum)*32/BytesPerInt, zmm ## regnum ## H_off
  95 #define DEF_OPMASK_OFFS(regnum)    opmask ## regnum ## _off = opmask_off + (regnum)*8/BytesPerInt,     opmask ## regnum ## H_off
  96 #define DEF_ZMM_UPPER_OFFS(regnum) zmm ## regnum ## _off = zmm_upper_off + (regnum-16)*64/BytesPerInt, zmm ## regnum ## H_off
  97   enum layout {
  98     fpu_state_off = frame::arg_reg_save_area_bytes/BytesPerInt,    // fxsave save area
  99     xmm_off       = fpu_state_off + XSAVE_AREA_BEGIN/BytesPerInt,  // offset in fxsave save area
 100     DEF_XMM_OFFS(0),
 101     DEF_XMM_OFFS(1),
 102     // 2..15 are implied in range usage
 103     ymm_off = xmm_off + (XSAVE_AREA_YMM_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt,
 104     DEF_YMM_OFFS(0),
 105     DEF_YMM_OFFS(1),
 106     r16_off = xmm_off + (XSAVE_AREA_EGPRS - XSAVE_AREA_BEGIN)/BytesPerInt,
 107     r16H_off,
 108     r17_off, r17H_off,
 109     r18_off, r18H_off,
 110     r19_off, r19H_off,
 111     r20_off, r20H_off,
 112     r21_off, r21H_off,
 113     r22_off, r22H_off,
 114     r23_off, r23H_off,
 115     r24_off, r24H_off,
 116     r25_off, r25H_off,
 117     r26_off, r26H_off,
 118     r27_off, r27H_off,
 119     r28_off, r28H_off,
 120     r29_off, r29H_off,
 121     r30_off, r30H_off,
 122     r31_off, r31H_off,
 123     opmask_off   = xmm_off + (XSAVE_AREA_OPMASK_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt,
 124     DEF_OPMASK_OFFS(0),
 125     DEF_OPMASK_OFFS(1),
 126     // 2..7 are implied in range usage
 127     zmm_off = xmm_off + (XSAVE_AREA_ZMM_BEGIN - XSAVE_AREA_BEGIN)/BytesPerInt,
 128     DEF_ZMM_OFFS(0),
 129     DEF_ZMM_OFFS(1),
 130     zmm_upper_off = xmm_off + (XSAVE_AREA_UPPERBANK - XSAVE_AREA_BEGIN)/BytesPerInt,
 131     DEF_ZMM_UPPER_OFFS(16),
 132     DEF_ZMM_UPPER_OFFS(17),
 133     // 18..31 are implied in range usage
 134     fpu_state_end = fpu_state_off + ((FPUStateSizeInWords-1)*wordSize / BytesPerInt),
 135     fpu_stateH_end,
 136     r15_off, r15H_off,
 137     r14_off, r14H_off,
 138     r13_off, r13H_off,
 139     r12_off, r12H_off,
 140     r11_off, r11H_off,
 141     r10_off, r10H_off,
 142     r9_off,  r9H_off,
 143     r8_off,  r8H_off,
 144     rdi_off, rdiH_off,
 145     rsi_off, rsiH_off,
 146     ignore_off, ignoreH_off,  // extra copy of rbp
 147     rsp_off, rspH_off,
 148     rbx_off, rbxH_off,
 149     rdx_off, rdxH_off,
 150     rcx_off, rcxH_off,
 151     rax_off, raxH_off,
 152     // 16-byte stack alignment fill word: see MacroAssembler::push/pop_IU_state
 153     align_off, alignH_off,
 154     flags_off, flagsH_off,
 155     // The frame sender code expects that rbp will be in the "natural" place and
 156     // will override any oopMap setting for it. We must therefore force the layout
 157     // so that it agrees with the frame sender code.
 158     rbp_off, rbpH_off,        // copy of rbp we will restore
 159     return_off, returnH_off,  // slot for return address
 160     reg_save_size             // size in compiler stack slots
 161   };
 162 
 163  public:
 164   static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_wide_vectors);
 165   static void restore_live_registers(MacroAssembler* masm, bool restore_wide_vectors = false);
 166 
 167   // Offsets into the register save area
 168   // Used by deoptimization when it is managing result register
 169   // values on its own
 170 
 171   static int rax_offset_in_bytes(void)    { return BytesPerInt * rax_off; }
 172   static int rdx_offset_in_bytes(void)    { return BytesPerInt * rdx_off; }
 173   static int rbx_offset_in_bytes(void)    { return BytesPerInt * rbx_off; }
 174   static int r15_offset_in_bytes(void)    { return BytesPerInt * r15_off; }
 175   static int xmm0_offset_in_bytes(void)   { return BytesPerInt * xmm0_off; }
 176   static int return_offset_in_bytes(void) { return BytesPerInt * return_off; }
 177 
 178   // During deoptimization only the result registers need to be restored,
 179   // all the other values have already been extracted.
 180   static void restore_result_registers(MacroAssembler* masm);
 181 };
 182 
 183 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words, bool save_wide_vectors) {
 184   int off = 0;
 185   int num_xmm_regs = XMMRegister::available_xmm_registers();
 186 #if COMPILER2_OR_JVMCI
 187   if (save_wide_vectors && UseAVX == 0) {
 188     save_wide_vectors = false; // vectors larger than 16 byte long are supported only with AVX
 189   }
 190   assert(!save_wide_vectors || MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported");
 191 #else
 192   save_wide_vectors = false; // vectors are generated only by C2 and JVMCI
 193 #endif
 194 
 195   // Always make the frame size 16-byte aligned, both vector and non vector stacks are always allocated
 196   int frame_size_in_bytes = align_up(reg_save_size*BytesPerInt, num_xmm_regs);
 197   // OopMap frame size is in compiler stack slots (jint's) not bytes or words
 198   int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
 199   // CodeBlob frame size is in words.
 200   int frame_size_in_words = frame_size_in_bytes / wordSize;
 201   *total_frame_words = frame_size_in_words;
 202 
 203   // Save registers, fpu state, and flags.
 204   // We assume caller has already pushed the return address onto the
 205   // stack, so rsp is 8-byte aligned here.
 206   // We push rpb twice in this sequence because we want the real rbp
 207   // to be under the return like a normal enter.
 208 
 209   __ enter();          // rsp becomes 16-byte aligned here
 210   __ pushf();
 211   // Make sure rsp stays 16-byte aligned
 212   __ subq(rsp, 8);
 213   // Push CPU state in multiple of 16 bytes
 214   __ save_legacy_gprs();
 215   __ push_FPU_state();
 216 
 217 
 218   // push cpu state handles this on EVEX enabled targets
 219   if (save_wide_vectors) {
 220     // Save upper half of YMM registers(0..15)
 221     int base_addr = XSAVE_AREA_YMM_BEGIN;
 222     for (int n = 0; n < 16; n++) {
 223       __ vextractf128_high(Address(rsp, base_addr+n*16), as_XMMRegister(n));
 224     }
 225     if (VM_Version::supports_evex()) {
 226       // Save upper half of ZMM registers(0..15)
 227       base_addr = XSAVE_AREA_ZMM_BEGIN;
 228       for (int n = 0; n < 16; n++) {
 229         __ vextractf64x4_high(Address(rsp, base_addr+n*32), as_XMMRegister(n));
 230       }
 231       // Save full ZMM registers(16..num_xmm_regs)
 232       base_addr = XSAVE_AREA_UPPERBANK;
 233       off = 0;
 234       int vector_len = Assembler::AVX_512bit;
 235       for (int n = 16; n < num_xmm_regs; n++) {
 236         __ evmovdqul(Address(rsp, base_addr+(off++*64)), as_XMMRegister(n), vector_len);
 237       }
 238 #if COMPILER2_OR_JVMCI
 239       base_addr = XSAVE_AREA_OPMASK_BEGIN;
 240       off = 0;
 241       for(int n = 0; n < KRegister::number_of_registers; n++) {
 242         __ kmov(Address(rsp, base_addr+(off++*8)), as_KRegister(n));
 243       }
 244 #endif
 245     }
 246   } else {
 247     if (VM_Version::supports_evex()) {
 248       // Save upper bank of XMM registers(16..31) for scalar or 16-byte vector usage
 249       int base_addr = XSAVE_AREA_UPPERBANK;
 250       off = 0;
 251       int vector_len = VM_Version::supports_avx512vl() ?  Assembler::AVX_128bit : Assembler::AVX_512bit;
 252       for (int n = 16; n < num_xmm_regs; n++) {
 253         __ evmovdqul(Address(rsp, base_addr+(off++*64)), as_XMMRegister(n), vector_len);
 254       }
 255 #if COMPILER2_OR_JVMCI
 256       base_addr = XSAVE_AREA_OPMASK_BEGIN;
 257       off = 0;
 258       for(int n = 0; n < KRegister::number_of_registers; n++) {
 259         __ kmov(Address(rsp, base_addr+(off++*8)), as_KRegister(n));
 260       }
 261 #endif
 262     }
 263   }
 264 
 265 #if COMPILER2_OR_JVMCI
 266   if (UseAPX) {
 267       int base_addr = XSAVE_AREA_EGPRS;
 268       off = 0;
 269       for (int n = 16; n < Register::number_of_registers; n++) {
 270         __ movq(Address(rsp, base_addr+(off++*8)), as_Register(n));
 271       }
 272   }
 273 #endif
 274 
 275   __ vzeroupper();
 276   if (frame::arg_reg_save_area_bytes != 0) {
 277     // Allocate argument register save area
 278     __ subptr(rsp, frame::arg_reg_save_area_bytes);
 279   }
 280 
 281   // Set an oopmap for the call site.  This oopmap will map all
 282   // oop-registers and debug-info registers as callee-saved.  This
 283   // will allow deoptimization at this safepoint to find all possible
 284   // debug-info recordings, as well as let GC find all oops.
 285 
 286   OopMapSet *oop_maps = new OopMapSet();
 287   OopMap* map = new OopMap(frame_size_in_slots, 0);
 288 
 289 #define STACK_OFFSET(x) VMRegImpl::stack2reg((x))
 290 
 291   map->set_callee_saved(STACK_OFFSET( rax_off ), rax->as_VMReg());
 292   map->set_callee_saved(STACK_OFFSET( rcx_off ), rcx->as_VMReg());
 293   map->set_callee_saved(STACK_OFFSET( rdx_off ), rdx->as_VMReg());
 294   map->set_callee_saved(STACK_OFFSET( rbx_off ), rbx->as_VMReg());
 295   // rbp location is known implicitly by the frame sender code, needs no oopmap
 296   // and the location where rbp was saved by is ignored
 297   map->set_callee_saved(STACK_OFFSET( rsi_off ), rsi->as_VMReg());
 298   map->set_callee_saved(STACK_OFFSET( rdi_off ), rdi->as_VMReg());
 299   map->set_callee_saved(STACK_OFFSET( r8_off  ), r8->as_VMReg());
 300   map->set_callee_saved(STACK_OFFSET( r9_off  ), r9->as_VMReg());
 301   map->set_callee_saved(STACK_OFFSET( r10_off ), r10->as_VMReg());
 302   map->set_callee_saved(STACK_OFFSET( r11_off ), r11->as_VMReg());
 303   map->set_callee_saved(STACK_OFFSET( r12_off ), r12->as_VMReg());
 304   map->set_callee_saved(STACK_OFFSET( r13_off ), r13->as_VMReg());
 305   map->set_callee_saved(STACK_OFFSET( r14_off ), r14->as_VMReg());
 306   map->set_callee_saved(STACK_OFFSET( r15_off ), r15->as_VMReg());
 307 
 308   if (UseAPX) {
 309     map->set_callee_saved(STACK_OFFSET( r16_off ), r16->as_VMReg());
 310     map->set_callee_saved(STACK_OFFSET( r17_off ), r17->as_VMReg());
 311     map->set_callee_saved(STACK_OFFSET( r18_off ), r18->as_VMReg());
 312     map->set_callee_saved(STACK_OFFSET( r19_off ), r19->as_VMReg());
 313     map->set_callee_saved(STACK_OFFSET( r20_off ), r20->as_VMReg());
 314     map->set_callee_saved(STACK_OFFSET( r21_off ), r21->as_VMReg());
 315     map->set_callee_saved(STACK_OFFSET( r22_off ), r22->as_VMReg());
 316     map->set_callee_saved(STACK_OFFSET( r23_off ), r23->as_VMReg());
 317     map->set_callee_saved(STACK_OFFSET( r24_off ), r24->as_VMReg());
 318     map->set_callee_saved(STACK_OFFSET( r25_off ), r25->as_VMReg());
 319     map->set_callee_saved(STACK_OFFSET( r26_off ), r26->as_VMReg());
 320     map->set_callee_saved(STACK_OFFSET( r27_off ), r27->as_VMReg());
 321     map->set_callee_saved(STACK_OFFSET( r28_off ), r28->as_VMReg());
 322     map->set_callee_saved(STACK_OFFSET( r29_off ), r29->as_VMReg());
 323     map->set_callee_saved(STACK_OFFSET( r30_off ), r30->as_VMReg());
 324     map->set_callee_saved(STACK_OFFSET( r31_off ), r31->as_VMReg());
 325   }
 326   // For both AVX and EVEX we will use the legacy FXSAVE area for xmm0..xmm15,
 327   // on EVEX enabled targets, we get it included in the xsave area
 328   off = xmm0_off;
 329   int delta = xmm1_off - off;
 330   for (int n = 0; n < 16; n++) {
 331     XMMRegister xmm_name = as_XMMRegister(n);
 332     map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg());
 333     off += delta;
 334   }
 335   if (UseAVX > 2) {
 336     // Obtain xmm16..xmm31 from the XSAVE area on EVEX enabled targets
 337     off = zmm16_off;
 338     delta = zmm17_off - off;
 339     for (int n = 16; n < num_xmm_regs; n++) {
 340       XMMRegister zmm_name = as_XMMRegister(n);
 341       map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg());
 342       off += delta;
 343     }
 344   }
 345 
 346 #if COMPILER2_OR_JVMCI
 347   if (save_wide_vectors) {
 348     // Save upper half of YMM registers(0..15)
 349     off = ymm0_off;
 350     delta = ymm1_off - ymm0_off;
 351     for (int n = 0; n < 16; n++) {
 352       XMMRegister ymm_name = as_XMMRegister(n);
 353       map->set_callee_saved(STACK_OFFSET(off), ymm_name->as_VMReg()->next(4));
 354       off += delta;
 355     }
 356     if (VM_Version::supports_evex()) {
 357       // Save upper half of ZMM registers(0..15)
 358       off = zmm0_off;
 359       delta = zmm1_off - zmm0_off;
 360       for (int n = 0; n < 16; n++) {
 361         XMMRegister zmm_name = as_XMMRegister(n);
 362         map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg()->next(8));
 363         off += delta;
 364       }
 365     }
 366   }
 367 #endif // COMPILER2_OR_JVMCI
 368 
 369   // %%% These should all be a waste but we'll keep things as they were for now
 370   if (true) {
 371     map->set_callee_saved(STACK_OFFSET( raxH_off ), rax->as_VMReg()->next());
 372     map->set_callee_saved(STACK_OFFSET( rcxH_off ), rcx->as_VMReg()->next());
 373     map->set_callee_saved(STACK_OFFSET( rdxH_off ), rdx->as_VMReg()->next());
 374     map->set_callee_saved(STACK_OFFSET( rbxH_off ), rbx->as_VMReg()->next());
 375     // rbp location is known implicitly by the frame sender code, needs no oopmap
 376     map->set_callee_saved(STACK_OFFSET( rsiH_off ), rsi->as_VMReg()->next());
 377     map->set_callee_saved(STACK_OFFSET( rdiH_off ), rdi->as_VMReg()->next());
 378     map->set_callee_saved(STACK_OFFSET( r8H_off  ), r8->as_VMReg()->next());
 379     map->set_callee_saved(STACK_OFFSET( r9H_off  ), r9->as_VMReg()->next());
 380     map->set_callee_saved(STACK_OFFSET( r10H_off ), r10->as_VMReg()->next());
 381     map->set_callee_saved(STACK_OFFSET( r11H_off ), r11->as_VMReg()->next());
 382     map->set_callee_saved(STACK_OFFSET( r12H_off ), r12->as_VMReg()->next());
 383     map->set_callee_saved(STACK_OFFSET( r13H_off ), r13->as_VMReg()->next());
 384     map->set_callee_saved(STACK_OFFSET( r14H_off ), r14->as_VMReg()->next());
 385     map->set_callee_saved(STACK_OFFSET( r15H_off ), r15->as_VMReg()->next());
 386     if (UseAPX) {
 387       map->set_callee_saved(STACK_OFFSET( r16H_off ), r16->as_VMReg()->next());
 388       map->set_callee_saved(STACK_OFFSET( r17H_off ), r17->as_VMReg()->next());
 389       map->set_callee_saved(STACK_OFFSET( r18H_off ), r18->as_VMReg()->next());
 390       map->set_callee_saved(STACK_OFFSET( r19H_off ), r19->as_VMReg()->next());
 391       map->set_callee_saved(STACK_OFFSET( r20H_off ), r20->as_VMReg()->next());
 392       map->set_callee_saved(STACK_OFFSET( r21H_off ), r21->as_VMReg()->next());
 393       map->set_callee_saved(STACK_OFFSET( r22H_off ), r22->as_VMReg()->next());
 394       map->set_callee_saved(STACK_OFFSET( r23H_off ), r23->as_VMReg()->next());
 395       map->set_callee_saved(STACK_OFFSET( r24H_off ), r24->as_VMReg()->next());
 396       map->set_callee_saved(STACK_OFFSET( r25H_off ), r25->as_VMReg()->next());
 397       map->set_callee_saved(STACK_OFFSET( r26H_off ), r26->as_VMReg()->next());
 398       map->set_callee_saved(STACK_OFFSET( r27H_off ), r27->as_VMReg()->next());
 399       map->set_callee_saved(STACK_OFFSET( r28H_off ), r28->as_VMReg()->next());
 400       map->set_callee_saved(STACK_OFFSET( r29H_off ), r29->as_VMReg()->next());
 401       map->set_callee_saved(STACK_OFFSET( r30H_off ), r30->as_VMReg()->next());
 402       map->set_callee_saved(STACK_OFFSET( r31H_off ), r31->as_VMReg()->next());
 403     }
 404     // For both AVX and EVEX we will use the legacy FXSAVE area for xmm0..xmm15,
 405     // on EVEX enabled targets, we get it included in the xsave area
 406     off = xmm0H_off;
 407     delta = xmm1H_off - off;
 408     for (int n = 0; n < 16; n++) {
 409       XMMRegister xmm_name = as_XMMRegister(n);
 410       map->set_callee_saved(STACK_OFFSET(off), xmm_name->as_VMReg()->next());
 411       off += delta;
 412     }
 413     if (UseAVX > 2) {
 414       // Obtain xmm16..xmm31 from the XSAVE area on EVEX enabled targets
 415       off = zmm16H_off;
 416       delta = zmm17H_off - off;
 417       for (int n = 16; n < num_xmm_regs; n++) {
 418         XMMRegister zmm_name = as_XMMRegister(n);
 419         map->set_callee_saved(STACK_OFFSET(off), zmm_name->as_VMReg()->next());
 420         off += delta;
 421       }
 422     }
 423   }
 424 
 425   return map;
 426 }
 427 
 428 void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_wide_vectors) {
 429   int num_xmm_regs = XMMRegister::available_xmm_registers();
 430   if (frame::arg_reg_save_area_bytes != 0) {
 431     // Pop arg register save area
 432     __ addptr(rsp, frame::arg_reg_save_area_bytes);
 433   }
 434 
 435 #if COMPILER2_OR_JVMCI
 436   if (restore_wide_vectors) {
 437     assert(UseAVX > 0, "Vectors larger than 16 byte long are supported only with AVX");
 438     assert(MaxVectorSize <= 64, "Only up to 64 byte long vectors are supported");
 439   }
 440 #else
 441   assert(!restore_wide_vectors, "vectors are generated only by C2");
 442 #endif
 443 
 444   __ vzeroupper();
 445 
 446   // On EVEX enabled targets everything is handled in pop fpu state
 447   if (restore_wide_vectors) {
 448     // Restore upper half of YMM registers (0..15)
 449     int base_addr = XSAVE_AREA_YMM_BEGIN;
 450     for (int n = 0; n < 16; n++) {
 451       __ vinsertf128_high(as_XMMRegister(n), Address(rsp, base_addr+n*16));
 452     }
 453     if (VM_Version::supports_evex()) {
 454       // Restore upper half of ZMM registers (0..15)
 455       base_addr = XSAVE_AREA_ZMM_BEGIN;
 456       for (int n = 0; n < 16; n++) {
 457         __ vinsertf64x4_high(as_XMMRegister(n), Address(rsp, base_addr+n*32));
 458       }
 459       // Restore full ZMM registers(16..num_xmm_regs)
 460       base_addr = XSAVE_AREA_UPPERBANK;
 461       int vector_len = Assembler::AVX_512bit;
 462       int off = 0;
 463       for (int n = 16; n < num_xmm_regs; n++) {
 464         __ evmovdqul(as_XMMRegister(n), Address(rsp, base_addr+(off++*64)), vector_len);
 465       }
 466 #if COMPILER2_OR_JVMCI
 467       base_addr = XSAVE_AREA_OPMASK_BEGIN;
 468       off = 0;
 469       for (int n = 0; n < KRegister::number_of_registers; n++) {
 470         __ kmov(as_KRegister(n), Address(rsp, base_addr+(off++*8)));
 471       }
 472 #endif
 473     }
 474   } else {
 475     if (VM_Version::supports_evex()) {
 476       // Restore upper bank of XMM registers(16..31) for scalar or 16-byte vector usage
 477       int base_addr = XSAVE_AREA_UPPERBANK;
 478       int off = 0;
 479       int vector_len = VM_Version::supports_avx512vl() ?  Assembler::AVX_128bit : Assembler::AVX_512bit;
 480       for (int n = 16; n < num_xmm_regs; n++) {
 481         __ evmovdqul(as_XMMRegister(n), Address(rsp, base_addr+(off++*64)), vector_len);
 482       }
 483 #if COMPILER2_OR_JVMCI
 484       base_addr = XSAVE_AREA_OPMASK_BEGIN;
 485       off = 0;
 486       for (int n = 0; n < KRegister::number_of_registers; n++) {
 487         __ kmov(as_KRegister(n), Address(rsp, base_addr+(off++*8)));
 488       }
 489 #endif
 490     }
 491   }
 492 
 493 #if COMPILER2_OR_JVMCI
 494   if (UseAPX) {
 495     int base_addr = XSAVE_AREA_EGPRS;
 496     int off = 0;
 497     for (int n = 16; n < Register::number_of_registers; n++) {
 498       __ movq(as_Register(n), Address(rsp, base_addr+(off++*8)));
 499     }
 500   }
 501 #endif
 502 
 503   // Recover CPU state
 504   __ pop_FPU_state();
 505   __ restore_legacy_gprs();
 506   __ addq(rsp, 8);
 507   __ popf();
 508   // Get the rbp described implicitly by the calling convention (no oopMap)
 509   __ pop(rbp);
 510 }
 511 
 512 void RegisterSaver::restore_result_registers(MacroAssembler* masm) {
 513 
 514   // Just restore result register. Only used by deoptimization. By
 515   // now any callee save register that needs to be restored to a c2
 516   // caller of the deoptee has been extracted into the vframeArray
 517   // and will be stuffed into the c2i adapter we create for later
 518   // restoration so only result registers need to be restored here.
 519 
 520   // Restore fp result register
 521   __ movdbl(xmm0, Address(rsp, xmm0_offset_in_bytes()));
 522   // Restore integer result register
 523   __ movptr(rax, Address(rsp, rax_offset_in_bytes()));
 524   __ movptr(rdx, Address(rsp, rdx_offset_in_bytes()));
 525 
 526   // Pop all of the register save are off the stack except the return address
 527   __ addptr(rsp, return_offset_in_bytes());
 528 }
 529 
 530 // Is vector's size (in bytes) bigger than a size saved by default?
 531 // 16 bytes XMM registers are saved by default using fxsave/fxrstor instructions.
 532 bool SharedRuntime::is_wide_vector(int size) {
 533   return size > 16;
 534 }
 535 
 536 // ---------------------------------------------------------------------------
 537 // Read the array of BasicTypes from a signature, and compute where the
 538 // arguments should go.  Values in the VMRegPair regs array refer to 4-byte
 539 // quantities.  Values less than VMRegImpl::stack0 are registers, those above
 540 // refer to 4-byte stack slots.  All stack slots are based off of the stack pointer
 541 // as framesizes are fixed.
 542 // VMRegImpl::stack0 refers to the first slot 0(sp).
 543 // and VMRegImpl::stack0+1 refers to the memory word 4-byes higher.
 544 // Register up to Register::number_of_registers are the 64-bit
 545 // integer registers.
 546 
 547 // Note: the INPUTS in sig_bt are in units of Java argument words, which are
 548 // either 32-bit or 64-bit depending on the build.  The OUTPUTS are in 32-bit
 549 // units regardless of build. Of course for i486 there is no 64 bit build
 550 
 551 // The Java calling convention is a "shifted" version of the C ABI.
 552 // By skipping the first C ABI register we can call non-static jni methods
 553 // with small numbers of arguments without having to shuffle the arguments
 554 // at all. Since we control the java ABI we ought to at least get some
 555 // advantage out of it.
 556 
 557 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
 558                                            VMRegPair *regs,
 559                                            int total_args_passed) {
 560 
 561   // Create the mapping between argument positions and
 562   // registers.
 563   static const Register INT_ArgReg[Argument::n_int_register_parameters_j] = {
 564     j_rarg0, j_rarg1, j_rarg2, j_rarg3, j_rarg4, j_rarg5
 565   };
 566   static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_j] = {
 567     j_farg0, j_farg1, j_farg2, j_farg3,
 568     j_farg4, j_farg5, j_farg6, j_farg7
 569   };
 570 
 571 
 572   uint int_args = 0;
 573   uint fp_args = 0;
 574   uint stk_args = 0;
 575 
 576   for (int i = 0; i < total_args_passed; i++) {
 577     switch (sig_bt[i]) {
 578     case T_BOOLEAN:
 579     case T_CHAR:
 580     case T_BYTE:
 581     case T_SHORT:
 582     case T_INT:
 583       if (int_args < Argument::n_int_register_parameters_j) {
 584         regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
 585       } else {
 586         stk_args = align_up(stk_args, 2);
 587         regs[i].set1(VMRegImpl::stack2reg(stk_args));
 588         stk_args += 1;
 589       }
 590       break;
 591     case T_VOID:
 592       // halves of T_LONG or T_DOUBLE
 593       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 594       regs[i].set_bad();
 595       break;
 596     case T_LONG:
 597       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 598       // fall through
 599     case T_OBJECT:
 600     case T_ARRAY:
 601     case T_ADDRESS:
 602       if (int_args < Argument::n_int_register_parameters_j) {
 603         regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
 604       } else {
 605         stk_args = align_up(stk_args, 2);
 606         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 607         stk_args += 2;
 608       }
 609       break;
 610     case T_FLOAT:
 611       if (fp_args < Argument::n_float_register_parameters_j) {
 612         regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
 613       } else {
 614         stk_args = align_up(stk_args, 2);
 615         regs[i].set1(VMRegImpl::stack2reg(stk_args));
 616         stk_args += 1;
 617       }
 618       break;
 619     case T_DOUBLE:
 620       assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
 621       if (fp_args < Argument::n_float_register_parameters_j) {
 622         regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
 623       } else {
 624         stk_args = align_up(stk_args, 2);
 625         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 626         stk_args += 2;
 627       }
 628       break;
 629     default:
 630       ShouldNotReachHere();
 631       break;
 632     }
 633   }
 634 
 635   return stk_args;
 636 }
 637 
 638 // Same as java_calling_convention() but for multiple return
 639 // values. There's no way to store them on the stack so if we don't
 640 // have enough registers, multiple values can't be returned.
 641 const uint SharedRuntime::java_return_convention_max_int = Argument::n_int_register_parameters_j+1;
 642 const uint SharedRuntime::java_return_convention_max_float = Argument::n_float_register_parameters_j;
 643 int SharedRuntime::java_return_convention(const BasicType *sig_bt,
 644                                           VMRegPair *regs,
 645                                           int total_args_passed) {
 646   // Create the mapping between argument positions and
 647   // registers.
 648   static const Register INT_ArgReg[java_return_convention_max_int] = {
 649     rax, j_rarg5, j_rarg4, j_rarg3, j_rarg2, j_rarg1, j_rarg0
 650   };
 651   static const XMMRegister FP_ArgReg[java_return_convention_max_float] = {
 652     j_farg0, j_farg1, j_farg2, j_farg3,
 653     j_farg4, j_farg5, j_farg6, j_farg7
 654   };
 655 
 656 
 657   uint int_args = 0;
 658   uint fp_args = 0;
 659 
 660   for (int i = 0; i < total_args_passed; i++) {
 661     switch (sig_bt[i]) {
 662     case T_BOOLEAN:
 663     case T_CHAR:
 664     case T_BYTE:
 665     case T_SHORT:
 666     case T_INT:
 667       if (int_args < Argument::n_int_register_parameters_j+1) {
 668         regs[i].set1(INT_ArgReg[int_args]->as_VMReg());
 669         int_args++;
 670       } else {
 671         return -1;
 672       }
 673       break;
 674     case T_VOID:
 675       // halves of T_LONG or T_DOUBLE
 676       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 677       regs[i].set_bad();
 678       break;
 679     case T_LONG:
 680       assert(sig_bt[i + 1] == T_VOID, "expecting half");
 681       // fall through
 682     case T_OBJECT:
 683     case T_ARRAY:
 684     case T_ADDRESS:
 685     case T_METADATA:
 686       if (int_args < Argument::n_int_register_parameters_j+1) {
 687         regs[i].set2(INT_ArgReg[int_args]->as_VMReg());
 688         int_args++;
 689       } else {
 690         return -1;
 691       }
 692       break;
 693     case T_FLOAT:
 694       if (fp_args < Argument::n_float_register_parameters_j) {
 695         regs[i].set1(FP_ArgReg[fp_args]->as_VMReg());
 696         fp_args++;
 697       } else {
 698         return -1;
 699       }
 700       break;
 701     case T_DOUBLE:
 702       assert(sig_bt[i + 1] == T_VOID, "expecting half");
 703       if (fp_args < Argument::n_float_register_parameters_j) {
 704         regs[i].set2(FP_ArgReg[fp_args]->as_VMReg());
 705         fp_args++;
 706       } else {
 707         return -1;
 708       }
 709       break;
 710     default:
 711       ShouldNotReachHere();
 712       break;
 713     }
 714   }
 715 
 716   return int_args + fp_args;
 717 }
 718 
 719 // Patch the callers callsite with entry to compiled code if it exists.
 720 static void patch_callers_callsite(MacroAssembler *masm) {
 721   Label L;
 722   __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD);
 723   __ jcc(Assembler::equal, L);
 724 
 725   // Save the current stack pointer
 726   __ mov(r13, rsp);
 727   // Schedule the branch target address early.
 728   // Call into the VM to patch the caller, then jump to compiled callee
 729   // rax isn't live so capture return address while we easily can
 730   __ movptr(rax, Address(rsp, 0));
 731 
 732   // align stack so push_CPU_state doesn't fault
 733   __ andptr(rsp, -(StackAlignmentInBytes));
 734   __ push_CPU_state();
 735   __ vzeroupper();
 736   // VM needs caller's callsite
 737   // VM needs target method
 738   // This needs to be a long call since we will relocate this adapter to
 739   // the codeBuffer and it may not reach
 740 
 741   // Allocate argument register save area
 742   if (frame::arg_reg_save_area_bytes != 0) {
 743     __ subptr(rsp, frame::arg_reg_save_area_bytes);
 744   }
 745   __ mov(c_rarg0, rbx);
 746   __ mov(c_rarg1, rax);
 747   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 748 
 749   // De-allocate argument register save area
 750   if (frame::arg_reg_save_area_bytes != 0) {
 751     __ addptr(rsp, frame::arg_reg_save_area_bytes);
 752   }
 753 
 754   __ vzeroupper();
 755   __ pop_CPU_state();
 756   // restore sp
 757   __ mov(rsp, r13);
 758   __ bind(L);
 759 }
 760 
 761 // For each inline type argument, sig includes the list of fields of
 762 // the inline type. This utility function computes the number of
 763 // arguments for the call if inline types are passed by reference (the
 764 // calling convention the interpreter expects).
 765 static int compute_total_args_passed_int(const GrowableArray<SigEntry>* sig_extended) {
 766   int total_args_passed = 0;
 767   if (InlineTypePassFieldsAsArgs) {
 768     for (int i = 0; i < sig_extended->length(); i++) {
 769       BasicType bt = sig_extended->at(i)._bt;
 770       if (bt == T_METADATA) {
 771         // In sig_extended, an inline type argument starts with:
 772         // T_METADATA, followed by the types of the fields of the
 773         // inline type and T_VOID to mark the end of the value
 774         // type. Inline types are flattened so, for instance, in the
 775         // case of an inline type with an int field and an inline type
 776         // field that itself has 2 fields, an int and a long:
 777         // T_METADATA T_INT T_METADATA T_INT T_LONG T_VOID (second
 778         // slot for the T_LONG) T_VOID (inner inline type) T_VOID
 779         // (outer inline type)
 780         total_args_passed++;
 781         int vt = 1;
 782         do {
 783           i++;
 784           BasicType bt = sig_extended->at(i)._bt;
 785           BasicType prev_bt = sig_extended->at(i-1)._bt;
 786           if (bt == T_METADATA) {
 787             vt++;
 788           } else if (bt == T_VOID &&
 789                      prev_bt != T_LONG &&
 790                      prev_bt != T_DOUBLE) {
 791             vt--;
 792           }
 793         } while (vt != 0);
 794       } else {
 795         total_args_passed++;
 796       }
 797     }
 798   } else {
 799     total_args_passed = sig_extended->length();
 800   }
 801   return total_args_passed;
 802 }
 803 
 804 
 805 static void gen_c2i_adapter_helper(MacroAssembler* masm,
 806                                    BasicType bt,
 807                                    BasicType prev_bt,
 808                                    size_t size_in_bytes,
 809                                    const VMRegPair& reg_pair,
 810                                    const Address& to,
 811                                    int extraspace,
 812                                    bool is_oop) {
 813   if (bt == T_VOID) {
 814     assert(prev_bt == T_LONG || prev_bt == T_DOUBLE, "missing half");
 815     return;
 816   }
 817 
 818   // Say 4 args:
 819   // i   st_off
 820   // 0   32 T_LONG
 821   // 1   24 T_VOID
 822   // 2   16 T_OBJECT
 823   // 3    8 T_BOOL
 824   // -    0 return address
 825   //
 826   // However to make thing extra confusing. Because we can fit a long/double in
 827   // a single slot on a 64 bt vm and it would be silly to break them up, the interpreter
 828   // leaves one slot empty and only stores to a single slot. In this case the
 829   // slot that is occupied is the T_VOID slot. See I said it was confusing.
 830 
 831   bool wide = (size_in_bytes == wordSize);
 832   VMReg r_1 = reg_pair.first();
 833   VMReg r_2 = reg_pair.second();
 834   assert(r_2->is_valid() == wide, "invalid size");
 835   if (!r_1->is_valid()) {
 836     assert(!r_2->is_valid(), "must be invalid");
 837     return;
 838   }
 839 
 840   if (!r_1->is_XMMRegister()) {
 841     Register val = rax;
 842     if (r_1->is_stack()) {
 843       int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 844       __ load_sized_value(val, Address(rsp, ld_off), size_in_bytes, /* is_signed */ false);
 845     } else {
 846       val = r_1->as_Register();
 847     }
 848     assert_different_registers(to.base(), val, rscratch1);
 849     if (is_oop) {
 850       __ push(r13);
 851       __ push(rbx);
 852       __ store_heap_oop(to, val, rscratch1, r13, rbx, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
 853       __ pop(rbx);
 854       __ pop(r13);
 855     } else {
 856       __ store_sized_value(to, val, size_in_bytes);
 857     }
 858   } else {
 859     if (wide) {
 860       __ movdbl(to, r_1->as_XMMRegister());
 861     } else {
 862       __ movflt(to, r_1->as_XMMRegister());
 863     }
 864   }
 865 }
 866 
 867 static void gen_c2i_adapter(MacroAssembler *masm,
 868                             const GrowableArray<SigEntry>* sig_extended,
 869                             const VMRegPair *regs,
 870                             bool requires_clinit_barrier,
 871                             address& c2i_no_clinit_check_entry,
 872                             Label& skip_fixup,
 873                             address start,
 874                             OopMapSet* oop_maps,
 875                             int& frame_complete,
 876                             int& frame_size_in_words,
 877                             bool alloc_inline_receiver) {
 878   if (requires_clinit_barrier && VM_Version::supports_fast_class_init_checks()) {
 879     Label L_skip_barrier;
 880     Register method = rbx;
 881 
 882     { // Bypass the barrier for non-static methods
 883       Register flags = rscratch1;
 884       __ load_unsigned_short(flags, Address(method, Method::access_flags_offset()));
 885       __ testl(flags, JVM_ACC_STATIC);
 886       __ jcc(Assembler::zero, L_skip_barrier); // non-static
 887     }
 888 
 889     Register klass = rscratch1;
 890     __ load_method_holder(klass, method);
 891     __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
 892 
 893     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
 894 
 895     __ bind(L_skip_barrier);
 896     c2i_no_clinit_check_entry = __ pc();
 897   }
 898 
 899   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 900   bs->c2i_entry_barrier(masm);
 901 
 902   // Before we get into the guts of the C2I adapter, see if we should be here
 903   // at all.  We've come from compiled code and are attempting to jump to the
 904   // interpreter, which means the caller made a static call to get here
 905   // (vcalls always get a compiled target if there is one).  Check for a
 906   // compiled target.  If there is one, we need to patch the caller's call.
 907   patch_callers_callsite(masm);
 908 
 909   __ bind(skip_fixup);
 910 
 911   if (InlineTypePassFieldsAsArgs) {
 912     // Is there an inline type argument?
 913     bool has_inline_argument = false;
 914     for (int i = 0; i < sig_extended->length() && !has_inline_argument; i++) {
 915       has_inline_argument = (sig_extended->at(i)._bt == T_METADATA);
 916     }
 917     if (has_inline_argument) {
 918       // There is at least an inline type argument: we're coming from
 919       // compiled code so we have no buffers to back the inline types.
 920       // Allocate the buffers here with a runtime call.
 921       OopMap* map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_vectors*/ false);
 922 
 923       frame_complete = __ offset();
 924 
 925       __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1);
 926 
 927       __ mov(c_rarg0, r15_thread);
 928       __ mov(c_rarg1, rbx);
 929       __ mov64(c_rarg2, (int64_t)alloc_inline_receiver);
 930       __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::allocate_inline_types)));
 931 
 932       oop_maps->add_gc_map((int)(__ pc() - start), map);
 933       __ reset_last_Java_frame(false);
 934 
 935       RegisterSaver::restore_live_registers(masm);
 936 
 937       Label no_exception;
 938       __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD);
 939       __ jcc(Assembler::equal, no_exception);
 940 
 941       __ movptr(Address(r15_thread, JavaThread::vm_result_oop_offset()), NULL_WORD);
 942       __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
 943       __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 944 
 945       __ bind(no_exception);
 946 
 947       // We get an array of objects from the runtime call
 948       __ get_vm_result_oop(rscratch2); // Use rscratch2 (r11) as temporary because rscratch1 (r10) is trashed by movptr()
 949       __ get_vm_result_metadata(rbx); // TODO: required to keep the callee Method live?
 950     }
 951   }
 952 
 953   // Since all args are passed on the stack, total_args_passed *
 954   // Interpreter::stackElementSize is the space we need.
 955   int total_args_passed = compute_total_args_passed_int(sig_extended);
 956   assert(total_args_passed >= 0, "total_args_passed is %d", total_args_passed);
 957 
 958   int extraspace = (total_args_passed * Interpreter::stackElementSize);
 959 
 960   // stack is aligned, keep it that way
 961   // This is not currently needed or enforced by the interpreter, but
 962   // we might as well conform to the ABI.
 963   extraspace = align_up(extraspace, 2*wordSize);
 964 
 965   // set senderSP value
 966   __ lea(r13, Address(rsp, wordSize));
 967 
 968 #ifdef ASSERT
 969   __ check_stack_alignment(r13, "sender stack not aligned");
 970 #endif
 971   if (extraspace > 0) {
 972     // Pop the return address
 973     __ pop(rax);
 974 
 975     __ subptr(rsp, extraspace);
 976 
 977     // Push the return address
 978     __ push(rax);
 979 
 980     // Account for the return address location since we store it first rather
 981     // than hold it in a register across all the shuffling
 982     extraspace += wordSize;
 983   }
 984 
 985 #ifdef ASSERT
 986   __ check_stack_alignment(rsp, "callee stack not aligned", wordSize, rax);
 987 #endif
 988 
 989   // Now write the args into the outgoing interpreter space
 990 
 991   // next_arg_comp is the next argument from the compiler point of
 992   // view (inline type fields are passed in registers/on the stack). In
 993   // sig_extended, an inline type argument starts with: T_METADATA,
 994   // followed by the types of the fields of the inline type and T_VOID
 995   // to mark the end of the inline type. ignored counts the number of
 996   // T_METADATA/T_VOID. next_vt_arg is the next inline type argument:
 997   // used to get the buffer for that argument from the pool of buffers
 998   // we allocated above and want to pass to the
 999   // interpreter. next_arg_int is the next argument from the
1000   // interpreter point of view (inline types are passed by reference).
1001   for (int next_arg_comp = 0, ignored = 0, next_vt_arg = 0, next_arg_int = 0;
1002        next_arg_comp < sig_extended->length(); next_arg_comp++) {
1003     assert(ignored <= next_arg_comp, "shouldn't skip over more slots than there are arguments");
1004     assert(next_arg_int <= total_args_passed, "more arguments for the interpreter than expected?");
1005     BasicType bt = sig_extended->at(next_arg_comp)._bt;
1006     int st_off = (total_args_passed - next_arg_int) * Interpreter::stackElementSize;
1007     if (!InlineTypePassFieldsAsArgs || bt != T_METADATA) {
1008       int next_off = st_off - Interpreter::stackElementSize;
1009       const int offset = (bt == T_LONG || bt == T_DOUBLE) ? next_off : st_off;
1010       const VMRegPair reg_pair = regs[next_arg_comp-ignored];
1011       size_t size_in_bytes = reg_pair.second()->is_valid() ? 8 : 4;
1012       gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
1013                              size_in_bytes, reg_pair, Address(rsp, offset), extraspace, false);
1014       next_arg_int++;
1015 #ifdef ASSERT
1016       if (bt == T_LONG || bt == T_DOUBLE) {
1017         // Overwrite the unused slot with known junk
1018         __ mov64(rax, CONST64(0xdeadffffdeadaaaa));
1019         __ movptr(Address(rsp, st_off), rax);
1020       }
1021 #endif /* ASSERT */
1022     } else {
1023       ignored++;
1024       // get the buffer from the just allocated pool of buffers
1025       int index = arrayOopDesc::base_offset_in_bytes(T_OBJECT) + next_vt_arg * type2aelembytes(T_OBJECT);
1026       __ load_heap_oop(r14, Address(rscratch2, index));
1027       next_vt_arg++; next_arg_int++;
1028       int vt = 1;
1029       // write fields we get from compiled code in registers/stack
1030       // slots to the buffer: we know we are done with that inline type
1031       // argument when we hit the T_VOID that acts as an end of inline
1032       // type delimiter for this inline type. Inline types are flattened
1033       // so we might encounter embedded inline types. Each entry in
1034       // sig_extended contains a field offset in the buffer.
1035       Label L_null;
1036       do {
1037         next_arg_comp++;
1038         BasicType bt = sig_extended->at(next_arg_comp)._bt;
1039         BasicType prev_bt = sig_extended->at(next_arg_comp-1)._bt;
1040         if (bt == T_METADATA) {
1041           vt++;
1042           ignored++;
1043         } else if (bt == T_VOID &&
1044                    prev_bt != T_LONG &&
1045                    prev_bt != T_DOUBLE) {
1046           vt--;
1047           ignored++;
1048         } else {
1049           int off = sig_extended->at(next_arg_comp)._offset;
1050           if (off == -1) {
1051             // Nullable inline type argument, emit null check
1052             VMReg reg = regs[next_arg_comp-ignored].first();
1053             Label L_notNull;
1054             if (reg->is_stack()) {
1055               int ld_off = reg->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
1056               __ testb(Address(rsp, ld_off), 1);
1057             } else {
1058               __ testb(reg->as_Register(), 1);
1059             }
1060             __ jcc(Assembler::notZero, L_notNull);
1061             __ movptr(Address(rsp, st_off), 0);
1062             __ jmp(L_null);
1063             __ bind(L_notNull);
1064             continue;
1065           }
1066           assert(off > 0, "offset in object should be positive");
1067           size_t size_in_bytes = is_java_primitive(bt) ? type2aelembytes(bt) : wordSize;
1068           bool is_oop = is_reference_type(bt);
1069           gen_c2i_adapter_helper(masm, bt, next_arg_comp > 0 ? sig_extended->at(next_arg_comp-1)._bt : T_ILLEGAL,
1070                                  size_in_bytes, regs[next_arg_comp-ignored], Address(r14, off), extraspace, is_oop);
1071         }
1072       } while (vt != 0);
1073       // pass the buffer to the interpreter
1074       __ movptr(Address(rsp, st_off), r14);
1075       __ bind(L_null);
1076     }
1077   }
1078 
1079   // Schedule the branch target address early.
1080   __ movptr(rcx, Address(rbx, in_bytes(Method::interpreter_entry_offset())));
1081   __ jmp(rcx);
1082 }
1083 
1084 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
1085                                     int comp_args_on_stack,
1086                                     const GrowableArray<SigEntry>* sig,
1087                                     const VMRegPair *regs) {
1088 
1089   // Note: r13 contains the senderSP on entry. We must preserve it since
1090   // we may do a i2c -> c2i transition if we lose a race where compiled
1091   // code goes non-entrant while we get args ready.
1092   // In addition we use r13 to locate all the interpreter args as
1093   // we must align the stack to 16 bytes on an i2c entry else we
1094   // lose alignment we expect in all compiled code and register
1095   // save code can segv when fxsave instructions find improperly
1096   // aligned stack pointer.
1097 
1098   // Adapters can be frameless because they do not require the caller
1099   // to perform additional cleanup work, such as correcting the stack pointer.
1100   // An i2c adapter is frameless because the *caller* frame, which is interpreted,
1101   // routinely repairs its own stack pointer (from interpreter_frame_last_sp),
1102   // even if a callee has modified the stack pointer.
1103   // A c2i adapter is frameless because the *callee* frame, which is interpreted,
1104   // routinely repairs its caller's stack pointer (from sender_sp, which is set
1105   // up via the senderSP register).
1106   // In other words, if *either* the caller or callee is interpreted, we can
1107   // get the stack pointer repaired after a call.
1108   // This is why c2i and i2c adapters cannot be indefinitely composed.
1109   // In particular, if a c2i adapter were to somehow call an i2c adapter,
1110   // both caller and callee would be compiled methods, and neither would
1111   // clean up the stack pointer changes performed by the two adapters.
1112   // If this happens, control eventually transfers back to the compiled
1113   // caller, but with an uncorrected stack, causing delayed havoc.
1114 
1115   // Must preserve original SP for loading incoming arguments because
1116   // we need to align the outgoing SP for compiled code.
1117   __ movptr(r11, rsp);
1118 
1119   // Pick up the return address
1120   __ pop(rax);
1121 
1122   // Convert 4-byte c2 stack slots to words.
1123   int comp_words_on_stack = align_up(comp_args_on_stack*VMRegImpl::stack_slot_size, wordSize)>>LogBytesPerWord;
1124 
1125   if (comp_args_on_stack) {
1126     __ subptr(rsp, comp_words_on_stack * wordSize);
1127   }
1128 
1129   // Ensure compiled code always sees stack at proper alignment
1130   __ andptr(rsp, -16);
1131 
1132   // push the return address and misalign the stack that youngest frame always sees
1133   // as far as the placement of the call instruction
1134   __ push(rax);
1135 
1136   // Put saved SP in another register
1137   const Register saved_sp = rax;
1138   __ movptr(saved_sp, r11);
1139 
1140   // Will jump to the compiled code just as if compiled code was doing it.
1141   // Pre-load the register-jump target early, to schedule it better.
1142   __ movptr(r11, Address(rbx, in_bytes(Method::from_compiled_inline_offset())));
1143 
1144 #if INCLUDE_JVMCI
1145   if (EnableJVMCI) {
1146     // check if this call should be routed towards a specific entry point
1147     __ cmpptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);
1148     Label no_alternative_target;
1149     __ jcc(Assembler::equal, no_alternative_target);
1150     __ movptr(r11, Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())));
1151     __ movptr(Address(r15_thread, in_bytes(JavaThread::jvmci_alternate_call_target_offset())), 0);
1152     __ bind(no_alternative_target);
1153   }
1154 #endif // INCLUDE_JVMCI
1155 
1156   int total_args_passed = sig->length();
1157 
1158   // Now generate the shuffle code.  Pick up all register args and move the
1159   // rest through the floating point stack top.
1160   for (int i = 0; i < total_args_passed; i++) {
1161     BasicType bt = sig->at(i)._bt;
1162     if (bt == T_VOID) {
1163       // Longs and doubles are passed in native word order, but misaligned
1164       // in the 32-bit build.
1165       BasicType prev_bt = (i > 0) ? sig->at(i-1)._bt : T_ILLEGAL;
1166       assert(i > 0 && (prev_bt == T_LONG || prev_bt == T_DOUBLE), "missing half");
1167       continue;
1168     }
1169 
1170     // Pick up 0, 1 or 2 words from SP+offset.
1171 
1172     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(),
1173             "scrambled load targets?");
1174     // Load in argument order going down.
1175     int ld_off = (total_args_passed - i)*Interpreter::stackElementSize;
1176     // Point to interpreter value (vs. tag)
1177     int next_off = ld_off - Interpreter::stackElementSize;
1178     //
1179     //
1180     //
1181     VMReg r_1 = regs[i].first();
1182     VMReg r_2 = regs[i].second();
1183     if (!r_1->is_valid()) {
1184       assert(!r_2->is_valid(), "");
1185       continue;
1186     }
1187     if (r_1->is_stack()) {
1188       // Convert stack slot to an SP offset (+ wordSize to account for return address )
1189       int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size + wordSize;
1190 
1191       // We can use r13 as a temp here because compiled code doesn't need r13 as an input
1192       // and if we end up going thru a c2i because of a miss a reasonable value of r13
1193       // will be generated.
1194       if (!r_2->is_valid()) {
1195         // sign extend???
1196         __ movl(r13, Address(saved_sp, ld_off));
1197         __ movptr(Address(rsp, st_off), r13);
1198       } else {
1199         //
1200         // We are using two optoregs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE
1201         // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case
1202         // So we must adjust where to pick up the data to match the interpreter.
1203         //
1204         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
1205         // are accessed as negative so LSW is at LOW address
1206 
1207         // ld_off is MSW so get LSW
1208         const int offset = (bt==T_LONG||bt==T_DOUBLE)?
1209                            next_off : ld_off;
1210         __ movq(r13, Address(saved_sp, offset));
1211         // st_off is LSW (i.e. reg.first())
1212         __ movq(Address(rsp, st_off), r13);
1213       }
1214     } else if (r_1->is_Register()) {  // Register argument
1215       Register r = r_1->as_Register();
1216       assert(r != rax, "must be different");
1217       if (r_2->is_valid()) {
1218         //
1219         // We are using two VMRegs. This can be either T_OBJECT, T_ADDRESS, T_LONG, or T_DOUBLE
1220         // the interpreter allocates two slots but only uses one for thr T_LONG or T_DOUBLE case
1221         // So we must adjust where to pick up the data to match the interpreter.
1222 
1223         const int offset = (bt==T_LONG||bt==T_DOUBLE)?
1224                            next_off : ld_off;
1225 
1226         // this can be a misaligned move
1227         __ movq(r, Address(saved_sp, offset));
1228       } else {
1229         // sign extend and use a full word?
1230         __ movl(r, Address(saved_sp, ld_off));
1231       }
1232     } else {
1233       if (!r_2->is_valid()) {
1234         __ movflt(r_1->as_XMMRegister(), Address(saved_sp, ld_off));
1235       } else {
1236         __ movdbl(r_1->as_XMMRegister(), Address(saved_sp, next_off));
1237       }
1238     }
1239   }
1240 
1241   __ push_cont_fastpath(); // Set JavaThread::_cont_fastpath to the sp of the oldest interpreted frame we know about
1242 
1243   // 6243940 We might end up in handle_wrong_method if
1244   // the callee is deoptimized as we race thru here. If that
1245   // happens we don't want to take a safepoint because the
1246   // caller frame will look interpreted and arguments are now
1247   // "compiled" so it is much better to make this transition
1248   // invisible to the stack walking code. Unfortunately if
1249   // we try and find the callee by normal means a safepoint
1250   // is possible. So we stash the desired callee in the thread
1251   // and the vm will find there should this case occur.
1252 
1253   __ movptr(Address(r15_thread, JavaThread::callee_target_offset()), rbx);
1254 
1255   // put Method* where a c2i would expect should we end up there
1256   // only needed because of c2 resolve stubs return Method* as a result in
1257   // rax
1258   __ mov(rax, rbx);
1259   __ jmp(r11);
1260 }
1261 
1262 static void gen_inline_cache_check(MacroAssembler *masm, Label& skip_fixup) {
1263   Register data = rax;
1264   __ ic_check(1 /* end_alignment */);
1265   __ movptr(rbx, Address(data, CompiledICData::speculated_method_offset()));
1266 
1267   // Method might have been compiled since the call site was patched to
1268   // interpreted if that is the case treat it as a miss so we can get
1269   // the call site corrected.
1270   __ cmpptr(Address(rbx, in_bytes(Method::code_offset())), NULL_WORD);
1271   __ jcc(Assembler::equal, skip_fixup);
1272   __ jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1273 }
1274 
1275 // ---------------------------------------------------------------
1276 void SharedRuntime::generate_i2c2i_adapters(MacroAssembler* masm,
1277                                             int comp_args_on_stack,
1278                                             const GrowableArray<SigEntry>* sig,
1279                                             const VMRegPair* regs,
1280                                             const GrowableArray<SigEntry>* sig_cc,
1281                                             const VMRegPair* regs_cc,
1282                                             const GrowableArray<SigEntry>* sig_cc_ro,
1283                                             const VMRegPair* regs_cc_ro,
1284                                             address entry_address[AdapterBlob::ENTRY_COUNT],
1285                                             AdapterBlob*& new_adapter,
1286                                             bool allocate_code_blob) {
1287   entry_address[AdapterBlob::I2C] = __ pc();
1288   gen_i2c_adapter(masm, comp_args_on_stack, sig, regs);
1289 
1290   // -------------------------------------------------------------------------
1291   // Generate a C2I adapter.  On entry we know rbx holds the Method* during calls
1292   // to the interpreter.  The args start out packed in the compiled layout.  They
1293   // need to be unpacked into the interpreter layout.  This will almost always
1294   // require some stack space.  We grow the current (compiled) stack, then repack
1295   // the args.  We  finally end in a jump to the generic interpreter entry point.
1296   // On exit from the interpreter, the interpreter will restore our SP (lest the
1297   // compiled code, which relies solely on SP and not RBP, get sick).
1298 
1299   entry_address[AdapterBlob::C2I_Unverified] = __ pc();
1300   entry_address[AdapterBlob::C2I_Unverified_Inline] = __ pc();
1301   Label skip_fixup;
1302 
1303   gen_inline_cache_check(masm, skip_fixup);
1304 
1305   OopMapSet* oop_maps = new OopMapSet();
1306   int frame_complete = CodeOffsets::frame_never_safe;
1307   int frame_size_in_words = 0;
1308 
1309   // Scalarized c2i adapter with non-scalarized receiver (i.e., don't pack receiver)
1310   entry_address[AdapterBlob::C2I_No_Clinit_Check] = nullptr;
1311   entry_address[AdapterBlob::C2I_Inline_RO] = __ pc();
1312   if (regs_cc != regs_cc_ro) {
1313     // No class init barrier needed because method is guaranteed to be non-static
1314     gen_c2i_adapter(masm, sig_cc_ro, regs_cc_ro, /* requires_clinit_barrier = */ false, entry_address[AdapterBlob::C2I_No_Clinit_Check],
1315                     skip_fixup, entry_address[AdapterBlob::I2C], oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ false);
1316     skip_fixup.reset();
1317   }
1318 
1319   // Scalarized c2i adapter
1320   entry_address[AdapterBlob::C2I]        = __ pc();
1321   entry_address[AdapterBlob::C2I_Inline] = __ pc();
1322   gen_c2i_adapter(masm, sig_cc, regs_cc, /* requires_clinit_barrier = */ true, entry_address[AdapterBlob::C2I_No_Clinit_Check],
1323                   skip_fixup, entry_address[AdapterBlob::I2C], oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ true);
1324 
1325   // Non-scalarized c2i adapter
1326   if (regs != regs_cc) {
1327     entry_address[AdapterBlob::C2I_Unverified_Inline] = __ pc();
1328     Label inline_entry_skip_fixup;
1329     gen_inline_cache_check(masm, inline_entry_skip_fixup);
1330 
1331     entry_address[AdapterBlob::C2I_Inline] = __ pc();
1332     gen_c2i_adapter(masm, sig, regs, /* requires_clinit_barrier = */ true, entry_address[AdapterBlob::C2I_No_Clinit_Check],
1333                     inline_entry_skip_fixup, entry_address[AdapterBlob::I2C], oop_maps, frame_complete, frame_size_in_words, /* alloc_inline_receiver = */ false);
1334   }
1335 
1336   // The c2i adapters might safepoint and trigger a GC. The caller must make sure that
1337   // the GC knows about the location of oop argument locations passed to the c2i adapter.
1338   if (allocate_code_blob) {
1339     bool caller_must_gc_arguments = (regs != regs_cc);
1340     int entry_offset[AdapterHandlerEntry::ENTRIES_COUNT];
1341     assert(AdapterHandlerEntry::ENTRIES_COUNT == 7, "sanity");
1342     AdapterHandlerLibrary::address_to_offset(entry_address, entry_offset);
1343     new_adapter = AdapterBlob::create(masm->code(), entry_offset, frame_complete, frame_size_in_words, oop_maps, caller_must_gc_arguments);
1344   }
1345 }
1346 
1347 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
1348                                          VMRegPair *regs,
1349                                          int total_args_passed) {
1350 
1351 // We return the amount of VMRegImpl stack slots we need to reserve for all
1352 // the arguments NOT counting out_preserve_stack_slots.
1353 
1354 // NOTE: These arrays will have to change when c1 is ported
1355 #ifdef _WIN64
1356     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1357       c_rarg0, c_rarg1, c_rarg2, c_rarg3
1358     };
1359     static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
1360       c_farg0, c_farg1, c_farg2, c_farg3
1361     };
1362 #else
1363     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
1364       c_rarg0, c_rarg1, c_rarg2, c_rarg3, c_rarg4, c_rarg5
1365     };
1366     static const XMMRegister FP_ArgReg[Argument::n_float_register_parameters_c] = {
1367       c_farg0, c_farg1, c_farg2, c_farg3,
1368       c_farg4, c_farg5, c_farg6, c_farg7
1369     };
1370 #endif // _WIN64
1371 
1372 
1373     uint int_args = 0;
1374     uint fp_args = 0;
1375     uint stk_args = 0; // inc by 2 each time
1376 
1377     for (int i = 0; i < total_args_passed; i++) {
1378       switch (sig_bt[i]) {
1379       case T_BOOLEAN:
1380       case T_CHAR:
1381       case T_BYTE:
1382       case T_SHORT:
1383       case T_INT:
1384         if (int_args < Argument::n_int_register_parameters_c) {
1385           regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
1386 #ifdef _WIN64
1387           fp_args++;
1388           // Allocate slots for callee to stuff register args the stack.
1389           stk_args += 2;
1390 #endif
1391         } else {
1392           regs[i].set1(VMRegImpl::stack2reg(stk_args));
1393           stk_args += 2;
1394         }
1395         break;
1396       case T_LONG:
1397         assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
1398         // fall through
1399       case T_OBJECT:
1400       case T_ARRAY:
1401       case T_ADDRESS:
1402       case T_METADATA:
1403         if (int_args < Argument::n_int_register_parameters_c) {
1404           regs[i].set2(INT_ArgReg[int_args++]->as_VMReg());
1405 #ifdef _WIN64
1406           fp_args++;
1407           stk_args += 2;
1408 #endif
1409         } else {
1410           regs[i].set2(VMRegImpl::stack2reg(stk_args));
1411           stk_args += 2;
1412         }
1413         break;
1414       case T_FLOAT:
1415         if (fp_args < Argument::n_float_register_parameters_c) {
1416           regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
1417 #ifdef _WIN64
1418           int_args++;
1419           // Allocate slots for callee to stuff register args the stack.
1420           stk_args += 2;
1421 #endif
1422         } else {
1423           regs[i].set1(VMRegImpl::stack2reg(stk_args));
1424           stk_args += 2;
1425         }
1426         break;
1427       case T_DOUBLE:
1428         assert((i + 1) < total_args_passed && sig_bt[i + 1] == T_VOID, "expecting half");
1429         if (fp_args < Argument::n_float_register_parameters_c) {
1430           regs[i].set2(FP_ArgReg[fp_args++]->as_VMReg());
1431 #ifdef _WIN64
1432           int_args++;
1433           // Allocate slots for callee to stuff register args the stack.
1434           stk_args += 2;
1435 #endif
1436         } else {
1437           regs[i].set2(VMRegImpl::stack2reg(stk_args));
1438           stk_args += 2;
1439         }
1440         break;
1441       case T_VOID: // Halves of longs and doubles
1442         assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
1443         regs[i].set_bad();
1444         break;
1445       default:
1446         ShouldNotReachHere();
1447         break;
1448       }
1449     }
1450 #ifdef _WIN64
1451   // windows abi requires that we always allocate enough stack space
1452   // for 4 64bit registers to be stored down.
1453   if (stk_args < 8) {
1454     stk_args = 8;
1455   }
1456 #endif // _WIN64
1457 
1458   return stk_args;
1459 }
1460 
1461 int SharedRuntime::vector_calling_convention(VMRegPair *regs,
1462                                              uint num_bits,
1463                                              uint total_args_passed) {
1464   assert(num_bits == 64 || num_bits == 128 || num_bits == 256 || num_bits == 512,
1465          "only certain vector sizes are supported for now");
1466 
1467   static const XMMRegister VEC_ArgReg[32] = {
1468      xmm0,  xmm1,  xmm2,  xmm3,  xmm4,  xmm5,  xmm6,  xmm7,
1469      xmm8,  xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15,
1470     xmm16, xmm17, xmm18, xmm19, xmm20, xmm21, xmm22, xmm23,
1471     xmm24, xmm25, xmm26, xmm27, xmm28, xmm29, xmm30, xmm31
1472   };
1473 
1474   uint stk_args = 0;
1475   uint fp_args = 0;
1476 
1477   for (uint i = 0; i < total_args_passed; i++) {
1478     VMReg vmreg = VEC_ArgReg[fp_args++]->as_VMReg();
1479     int next_val = num_bits == 64 ? 1 : (num_bits == 128 ? 3 : (num_bits  == 256 ? 7 : 15));
1480     regs[i].set_pair(vmreg->next(next_val), vmreg);
1481   }
1482 
1483   return stk_args;
1484 }
1485 
1486 void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1487   // We always ignore the frame_slots arg and just use the space just below frame pointer
1488   // which by this time is free to use
1489   switch (ret_type) {
1490   case T_FLOAT:
1491     __ movflt(Address(rbp, -wordSize), xmm0);
1492     break;
1493   case T_DOUBLE:
1494     __ movdbl(Address(rbp, -wordSize), xmm0);
1495     break;
1496   case T_VOID:  break;
1497   default: {
1498     __ movptr(Address(rbp, -wordSize), rax);
1499     }
1500   }
1501 }
1502 
1503 void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1504   // We always ignore the frame_slots arg and just use the space just below frame pointer
1505   // which by this time is free to use
1506   switch (ret_type) {
1507   case T_FLOAT:
1508     __ movflt(xmm0, Address(rbp, -wordSize));
1509     break;
1510   case T_DOUBLE:
1511     __ movdbl(xmm0, Address(rbp, -wordSize));
1512     break;
1513   case T_VOID:  break;
1514   default: {
1515     __ movptr(rax, Address(rbp, -wordSize));
1516     }
1517   }
1518 }
1519 
1520 static void save_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
1521     for ( int i = first_arg ; i < arg_count ; i++ ) {
1522       if (args[i].first()->is_Register()) {
1523         __ push(args[i].first()->as_Register());
1524       } else if (args[i].first()->is_XMMRegister()) {
1525         __ subptr(rsp, 2*wordSize);
1526         __ movdbl(Address(rsp, 0), args[i].first()->as_XMMRegister());
1527       }
1528     }
1529 }
1530 
1531 static void restore_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
1532     for ( int i = arg_count - 1 ; i >= first_arg ; i-- ) {
1533       if (args[i].first()->is_Register()) {
1534         __ pop(args[i].first()->as_Register());
1535       } else if (args[i].first()->is_XMMRegister()) {
1536         __ movdbl(args[i].first()->as_XMMRegister(), Address(rsp, 0));
1537         __ addptr(rsp, 2*wordSize);
1538       }
1539     }
1540 }
1541 
1542 static void verify_oop_args(MacroAssembler* masm,
1543                             const methodHandle& method,
1544                             const BasicType* sig_bt,
1545                             const VMRegPair* regs) {
1546   Register temp_reg = rbx;  // not part of any compiled calling seq
1547   if (VerifyOops) {
1548     for (int i = 0; i < method->size_of_parameters(); i++) {
1549       if (is_reference_type(sig_bt[i])) {
1550         VMReg r = regs[i].first();
1551         assert(r->is_valid(), "bad oop arg");
1552         if (r->is_stack()) {
1553           __ movptr(temp_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1554           __ verify_oop(temp_reg);
1555         } else {
1556           __ verify_oop(r->as_Register());
1557         }
1558       }
1559     }
1560   }
1561 }
1562 
1563 static void check_continuation_enter_argument(VMReg actual_vmreg,
1564                                               Register expected_reg,
1565                                               const char* name) {
1566   assert(!actual_vmreg->is_stack(), "%s cannot be on stack", name);
1567   assert(actual_vmreg->as_Register() == expected_reg,
1568          "%s is in unexpected register: %s instead of %s",
1569          name, actual_vmreg->as_Register()->name(), expected_reg->name());
1570 }
1571 
1572 
1573 //---------------------------- continuation_enter_setup ---------------------------
1574 //
1575 // Arguments:
1576 //   None.
1577 //
1578 // Results:
1579 //   rsp: pointer to blank ContinuationEntry
1580 //
1581 // Kills:
1582 //   rax
1583 //
1584 static OopMap* continuation_enter_setup(MacroAssembler* masm, int& stack_slots) {
1585   assert(ContinuationEntry::size() % VMRegImpl::stack_slot_size == 0, "");
1586   assert(in_bytes(ContinuationEntry::cont_offset())  % VMRegImpl::stack_slot_size == 0, "");
1587   assert(in_bytes(ContinuationEntry::chunk_offset()) % VMRegImpl::stack_slot_size == 0, "");
1588 
1589   stack_slots += checked_cast<int>(ContinuationEntry::size()) / wordSize;
1590   __ subptr(rsp, checked_cast<int32_t>(ContinuationEntry::size()));
1591 
1592   int frame_size = (checked_cast<int>(ContinuationEntry::size()) + wordSize) / VMRegImpl::stack_slot_size;
1593   OopMap* map = new OopMap(frame_size, 0);
1594 
1595   __ movptr(rax, Address(r15_thread, JavaThread::cont_entry_offset()));
1596   __ movptr(Address(rsp, ContinuationEntry::parent_offset()), rax);
1597   __ movptr(Address(r15_thread, JavaThread::cont_entry_offset()), rsp);
1598 
1599   return map;
1600 }
1601 
1602 //---------------------------- fill_continuation_entry ---------------------------
1603 //
1604 // Arguments:
1605 //   rsp: pointer to blank Continuation entry
1606 //   reg_cont_obj: pointer to the continuation
1607 //   reg_flags: flags
1608 //
1609 // Results:
1610 //   rsp: pointer to filled out ContinuationEntry
1611 //
1612 // Kills:
1613 //   rax
1614 //
1615 static void fill_continuation_entry(MacroAssembler* masm, Register reg_cont_obj, Register reg_flags) {
1616   assert_different_registers(rax, reg_cont_obj, reg_flags);
1617 #ifdef ASSERT
1618   __ movl(Address(rsp, ContinuationEntry::cookie_offset()), ContinuationEntry::cookie_value());
1619 #endif
1620   __ movptr(Address(rsp, ContinuationEntry::cont_offset()), reg_cont_obj);
1621   __ movl  (Address(rsp, ContinuationEntry::flags_offset()), reg_flags);
1622   __ movptr(Address(rsp, ContinuationEntry::chunk_offset()), 0);
1623   __ movl(Address(rsp, ContinuationEntry::argsize_offset()), 0);
1624   __ movl(Address(rsp, ContinuationEntry::pin_count_offset()), 0);
1625 
1626   __ movptr(rax, Address(r15_thread, JavaThread::cont_fastpath_offset()));
1627   __ movptr(Address(rsp, ContinuationEntry::parent_cont_fastpath_offset()), rax);
1628   __ movq(rax, Address(r15_thread, JavaThread::held_monitor_count_offset()));
1629   __ movq(Address(rsp, ContinuationEntry::parent_held_monitor_count_offset()), rax);
1630 
1631   __ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), 0);
1632   __ movq(Address(r15_thread, JavaThread::held_monitor_count_offset()), 0);
1633 }
1634 
1635 //---------------------------- continuation_enter_cleanup ---------------------------
1636 //
1637 // Arguments:
1638 //   rsp: pointer to the ContinuationEntry
1639 //
1640 // Results:
1641 //   rsp: pointer to the spilled rbp in the entry frame
1642 //
1643 // Kills:
1644 //   rbx
1645 //
1646 static void continuation_enter_cleanup(MacroAssembler* masm) {
1647 #ifdef ASSERT
1648   Label L_good_sp;
1649   __ cmpptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
1650   __ jcc(Assembler::equal, L_good_sp);
1651   __ stop("Incorrect rsp at continuation_enter_cleanup");
1652   __ bind(L_good_sp);
1653 #endif
1654   __ movptr(rbx, Address(rsp, ContinuationEntry::parent_cont_fastpath_offset()));
1655   __ movptr(Address(r15_thread, JavaThread::cont_fastpath_offset()), rbx);
1656 
1657   if (CheckJNICalls) {
1658     // Check if this is a virtual thread continuation
1659     Label L_skip_vthread_code;
1660     __ cmpl(Address(rsp, ContinuationEntry::flags_offset()), 0);
1661     __ jcc(Assembler::equal, L_skip_vthread_code);
1662 
1663     // If the held monitor count is > 0 and this vthread is terminating then
1664     // it failed to release a JNI monitor. So we issue the same log message
1665     // that JavaThread::exit does.
1666     __ cmpptr(Address(r15_thread, JavaThread::jni_monitor_count_offset()), 0);
1667     __ jcc(Assembler::equal, L_skip_vthread_code);
1668 
1669     // rax may hold an exception oop, save it before the call
1670     __ push(rax);
1671     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::log_jni_monitor_still_held));
1672     __ pop(rax);
1673 
1674     // For vthreads we have to explicitly zero the JNI monitor count of the carrier
1675     // on termination. The held count is implicitly zeroed below when we restore from
1676     // the parent held count (which has to be zero).
1677     __ movq(Address(r15_thread, JavaThread::jni_monitor_count_offset()), 0);
1678 
1679     __ bind(L_skip_vthread_code);
1680   }
1681 #ifdef ASSERT
1682   else {
1683     // Check if this is a virtual thread continuation
1684     Label L_skip_vthread_code;
1685     __ cmpl(Address(rsp, ContinuationEntry::flags_offset()), 0);
1686     __ jcc(Assembler::equal, L_skip_vthread_code);
1687 
1688     // See comment just above. If not checking JNI calls the JNI count is only
1689     // needed for assertion checking.
1690     __ movq(Address(r15_thread, JavaThread::jni_monitor_count_offset()), 0);
1691 
1692     __ bind(L_skip_vthread_code);
1693   }
1694 #endif
1695 
1696   __ movq(rbx, Address(rsp, ContinuationEntry::parent_held_monitor_count_offset()));
1697   __ movq(Address(r15_thread, JavaThread::held_monitor_count_offset()), rbx);
1698 
1699   __ movptr(rbx, Address(rsp, ContinuationEntry::parent_offset()));
1700   __ movptr(Address(r15_thread, JavaThread::cont_entry_offset()), rbx);
1701   __ addptr(rsp, checked_cast<int32_t>(ContinuationEntry::size()));
1702 }
1703 
1704 static void gen_continuation_enter(MacroAssembler* masm,
1705                                    const VMRegPair* regs,
1706                                    int& exception_offset,
1707                                    OopMapSet* oop_maps,
1708                                    int& frame_complete,
1709                                    int& stack_slots,
1710                                    int& interpreted_entry_offset,
1711                                    int& compiled_entry_offset) {
1712 
1713   // enterSpecial(Continuation c, boolean isContinue, boolean isVirtualThread)
1714   int pos_cont_obj   = 0;
1715   int pos_is_cont    = 1;
1716   int pos_is_virtual = 2;
1717 
1718   // The platform-specific calling convention may present the arguments in various registers.
1719   // To simplify the rest of the code, we expect the arguments to reside at these known
1720   // registers, and we additionally check the placement here in case calling convention ever
1721   // changes.
1722   Register reg_cont_obj   = c_rarg1;
1723   Register reg_is_cont    = c_rarg2;
1724   Register reg_is_virtual = c_rarg3;
1725 
1726   check_continuation_enter_argument(regs[pos_cont_obj].first(),   reg_cont_obj,   "Continuation object");
1727   check_continuation_enter_argument(regs[pos_is_cont].first(),    reg_is_cont,    "isContinue");
1728   check_continuation_enter_argument(regs[pos_is_virtual].first(), reg_is_virtual, "isVirtualThread");
1729 
1730   // Utility methods kill rax, make sure there are no collisions
1731   assert_different_registers(rax, reg_cont_obj, reg_is_cont, reg_is_virtual);
1732 
1733   AddressLiteral resolve(SharedRuntime::get_resolve_static_call_stub(),
1734                          relocInfo::static_call_type);
1735 
1736   address start = __ pc();
1737 
1738   Label L_thaw, L_exit;
1739 
1740   // i2i entry used at interp_only_mode only
1741   interpreted_entry_offset = __ pc() - start;
1742   {
1743 #ifdef ASSERT
1744     Label is_interp_only;
1745     __ cmpb(Address(r15_thread, JavaThread::interp_only_mode_offset()), 0);
1746     __ jcc(Assembler::notEqual, is_interp_only);
1747     __ stop("enterSpecial interpreter entry called when not in interp_only_mode");
1748     __ bind(is_interp_only);
1749 #endif
1750 
1751     __ pop(rax); // return address
1752     // Read interpreter arguments into registers (this is an ad-hoc i2c adapter)
1753     __ movptr(c_rarg1, Address(rsp, Interpreter::stackElementSize*2));
1754     __ movl(c_rarg2,   Address(rsp, Interpreter::stackElementSize*1));
1755     __ movl(c_rarg3,   Address(rsp, Interpreter::stackElementSize*0));
1756     __ andptr(rsp, -16); // Ensure compiled code always sees stack at proper alignment
1757     __ push(rax); // return address
1758     __ push_cont_fastpath();
1759 
1760     __ enter();
1761 
1762     stack_slots = 2; // will be adjusted in setup
1763     OopMap* map = continuation_enter_setup(masm, stack_slots);
1764     // The frame is complete here, but we only record it for the compiled entry, so the frame would appear unsafe,
1765     // but that's okay because at the very worst we'll miss an async sample, but we're in interp_only_mode anyway.
1766 
1767     __ verify_oop(reg_cont_obj);
1768 
1769     fill_continuation_entry(masm, reg_cont_obj, reg_is_virtual);
1770 
1771     // If continuation, call to thaw. Otherwise, resolve the call and exit.
1772     __ testptr(reg_is_cont, reg_is_cont);
1773     __ jcc(Assembler::notZero, L_thaw);
1774 
1775     // --- Resolve path
1776 
1777     // Make sure the call is patchable
1778     __ align(BytesPerWord, __ offset() + NativeCall::displacement_offset);
1779     // Emit stub for static call
1780     address stub = CompiledDirectCall::emit_to_interp_stub(masm, __ pc());
1781     if (stub == nullptr) {
1782       fatal("CodeCache is full at gen_continuation_enter");
1783     }
1784     __ call(resolve);
1785     oop_maps->add_gc_map(__ pc() - start, map);
1786     __ post_call_nop();
1787 
1788     __ jmp(L_exit);
1789   }
1790 
1791   // compiled entry
1792   __ align(CodeEntryAlignment);
1793   compiled_entry_offset = __ pc() - start;
1794   __ enter();
1795 
1796   stack_slots = 2; // will be adjusted in setup
1797   OopMap* map = continuation_enter_setup(masm, stack_slots);
1798 
1799   // Frame is now completed as far as size and linkage.
1800   frame_complete = __ pc() - start;
1801 
1802   __ verify_oop(reg_cont_obj);
1803 
1804   fill_continuation_entry(masm, reg_cont_obj, reg_is_virtual);
1805 
1806   // If isContinue, call to thaw. Otherwise, call Continuation.enter(Continuation c, boolean isContinue)
1807   __ testptr(reg_is_cont, reg_is_cont);
1808   __ jccb(Assembler::notZero, L_thaw);
1809 
1810   // --- call Continuation.enter(Continuation c, boolean isContinue)
1811 
1812   // Make sure the call is patchable
1813   __ align(BytesPerWord, __ offset() + NativeCall::displacement_offset);
1814 
1815   // Emit stub for static call
1816   address stub = CompiledDirectCall::emit_to_interp_stub(masm, __ pc());
1817   if (stub == nullptr) {
1818     fatal("CodeCache is full at gen_continuation_enter");
1819   }
1820 
1821   // The call needs to be resolved. There's a special case for this in
1822   // SharedRuntime::find_callee_info_helper() which calls
1823   // LinkResolver::resolve_continuation_enter() which resolves the call to
1824   // Continuation.enter(Continuation c, boolean isContinue).
1825   __ call(resolve);
1826 
1827   oop_maps->add_gc_map(__ pc() - start, map);
1828   __ post_call_nop();
1829 
1830   __ jmpb(L_exit);
1831 
1832   // --- Thawing path
1833 
1834   __ bind(L_thaw);
1835 
1836   ContinuationEntry::_thaw_call_pc_offset = __ pc() - start;
1837   __ call(RuntimeAddress(StubRoutines::cont_thaw()));
1838 
1839   ContinuationEntry::_return_pc_offset = __ pc() - start;
1840   oop_maps->add_gc_map(__ pc() - start, map->deep_copy());
1841   __ post_call_nop();
1842 
1843   // --- Normal exit (resolve/thawing)
1844 
1845   __ bind(L_exit);
1846   ContinuationEntry::_cleanup_offset = __ pc() - start;
1847   continuation_enter_cleanup(masm);
1848   __ pop(rbp);
1849   __ ret(0);
1850 
1851   // --- Exception handling path
1852 
1853   exception_offset = __ pc() - start;
1854 
1855   continuation_enter_cleanup(masm);
1856   __ pop(rbp);
1857 
1858   __ movptr(c_rarg0, r15_thread);
1859   __ movptr(c_rarg1, Address(rsp, 0)); // return address
1860 
1861   // rax still holds the original exception oop, save it before the call
1862   __ push(rax);
1863 
1864   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), 2);
1865   __ movptr(rbx, rax);
1866 
1867   // Continue at exception handler:
1868   //   rax: exception oop
1869   //   rbx: exception handler
1870   //   rdx: exception pc
1871   __ pop(rax);
1872   __ verify_oop(rax);
1873   __ pop(rdx);
1874   __ jmp(rbx);
1875 }
1876 
1877 static void gen_continuation_yield(MacroAssembler* masm,
1878                                    const VMRegPair* regs,
1879                                    OopMapSet* oop_maps,
1880                                    int& frame_complete,
1881                                    int& stack_slots,
1882                                    int& compiled_entry_offset) {
1883   enum layout {
1884     rbp_off,
1885     rbpH_off,
1886     return_off,
1887     return_off2,
1888     framesize // inclusive of return address
1889   };
1890   stack_slots = framesize /  VMRegImpl::slots_per_word;
1891   assert(stack_slots == 2, "recheck layout");
1892 
1893   address start = __ pc();
1894   compiled_entry_offset = __ pc() - start;
1895   __ enter();
1896   address the_pc = __ pc();
1897 
1898   frame_complete = the_pc - start;
1899 
1900   // This nop must be exactly at the PC we push into the frame info.
1901   // We use this nop for fast CodeBlob lookup, associate the OopMap
1902   // with it right away.
1903   __ post_call_nop();
1904   OopMap* map = new OopMap(framesize, 1);
1905   oop_maps->add_gc_map(frame_complete, map);
1906 
1907   __ set_last_Java_frame(rsp, rbp, the_pc, rscratch1);
1908   __ movptr(c_rarg0, r15_thread);
1909   __ movptr(c_rarg1, rsp);
1910   __ call_VM_leaf(Continuation::freeze_entry(), 2);
1911   __ reset_last_Java_frame(true);
1912 
1913   Label L_pinned;
1914 
1915   __ testptr(rax, rax);
1916   __ jcc(Assembler::notZero, L_pinned);
1917 
1918   __ movptr(rsp, Address(r15_thread, JavaThread::cont_entry_offset()));
1919   continuation_enter_cleanup(masm);
1920   __ pop(rbp);
1921   __ ret(0);
1922 
1923   __ bind(L_pinned);
1924 
1925   // Pinned, return to caller
1926 
1927   // handle pending exception thrown by freeze
1928   __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD);
1929   Label ok;
1930   __ jcc(Assembler::equal, ok);
1931   __ leave();
1932   __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
1933   __ bind(ok);
1934 
1935   __ leave();
1936   __ ret(0);
1937 }
1938 
1939 void SharedRuntime::continuation_enter_cleanup(MacroAssembler* masm) {
1940   ::continuation_enter_cleanup(masm);
1941 }
1942 
1943 static void gen_special_dispatch(MacroAssembler* masm,
1944                                  const methodHandle& method,
1945                                  const BasicType* sig_bt,
1946                                  const VMRegPair* regs) {
1947   verify_oop_args(masm, method, sig_bt, regs);
1948   vmIntrinsics::ID iid = method->intrinsic_id();
1949 
1950   // Now write the args into the outgoing interpreter space
1951   bool     has_receiver   = false;
1952   Register receiver_reg   = noreg;
1953   int      member_arg_pos = -1;
1954   Register member_reg     = noreg;
1955   int      ref_kind       = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
1956   if (ref_kind != 0) {
1957     member_arg_pos = method->size_of_parameters() - 1;  // trailing MemberName argument
1958     member_reg = rbx;  // known to be free at this point
1959     has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
1960   } else if (iid == vmIntrinsics::_invokeBasic) {
1961     has_receiver = true;
1962   } else if (iid == vmIntrinsics::_linkToNative) {
1963     member_arg_pos = method->size_of_parameters() - 1;  // trailing NativeEntryPoint argument
1964     member_reg = rbx;  // known to be free at this point
1965   } else {
1966     fatal("unexpected intrinsic id %d", vmIntrinsics::as_int(iid));
1967   }
1968 
1969   if (member_reg != noreg) {
1970     // Load the member_arg into register, if necessary.
1971     SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
1972     VMReg r = regs[member_arg_pos].first();
1973     if (r->is_stack()) {
1974       __ movptr(member_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1975     } else {
1976       // no data motion is needed
1977       member_reg = r->as_Register();
1978     }
1979   }
1980 
1981   if (has_receiver) {
1982     // Make sure the receiver is loaded into a register.
1983     assert(method->size_of_parameters() > 0, "oob");
1984     assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
1985     VMReg r = regs[0].first();
1986     assert(r->is_valid(), "bad receiver arg");
1987     if (r->is_stack()) {
1988       // Porting note:  This assumes that compiled calling conventions always
1989       // pass the receiver oop in a register.  If this is not true on some
1990       // platform, pick a temp and load the receiver from stack.
1991       fatal("receiver always in a register");
1992       receiver_reg = j_rarg0;  // known to be free at this point
1993       __ movptr(receiver_reg, Address(rsp, r->reg2stack() * VMRegImpl::stack_slot_size + wordSize));
1994     } else {
1995       // no data motion is needed
1996       receiver_reg = r->as_Register();
1997     }
1998   }
1999 
2000   // Figure out which address we are really jumping to:
2001   MethodHandles::generate_method_handle_dispatch(masm, iid,
2002                                                  receiver_reg, member_reg, /*for_compiler_entry:*/ true);
2003 }
2004 
2005 // ---------------------------------------------------------------------------
2006 // Generate a native wrapper for a given method.  The method takes arguments
2007 // in the Java compiled code convention, marshals them to the native
2008 // convention (handlizes oops, etc), transitions to native, makes the call,
2009 // returns to java state (possibly blocking), unhandlizes any result and
2010 // returns.
2011 //
2012 // Critical native functions are a shorthand for the use of
2013 // GetPrimtiveArrayCritical and disallow the use of any other JNI
2014 // functions.  The wrapper is expected to unpack the arguments before
2015 // passing them to the callee. Critical native functions leave the state _in_Java,
2016 // since they cannot stop for GC.
2017 // Some other parts of JNI setup are skipped like the tear down of the JNI handle
2018 // block and the check for pending exceptions it's impossible for them
2019 // to be thrown.
2020 //
2021 nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
2022                                                 const methodHandle& method,
2023                                                 int compile_id,
2024                                                 BasicType* in_sig_bt,
2025                                                 VMRegPair* in_regs,
2026                                                 BasicType ret_type) {
2027   if (method->is_continuation_native_intrinsic()) {
2028     int exception_offset = -1;
2029     OopMapSet* oop_maps = new OopMapSet();
2030     int frame_complete = -1;
2031     int stack_slots = -1;
2032     int interpreted_entry_offset = -1;
2033     int vep_offset = -1;
2034     if (method->is_continuation_enter_intrinsic()) {
2035       gen_continuation_enter(masm,
2036                              in_regs,
2037                              exception_offset,
2038                              oop_maps,
2039                              frame_complete,
2040                              stack_slots,
2041                              interpreted_entry_offset,
2042                              vep_offset);
2043     } else if (method->is_continuation_yield_intrinsic()) {
2044       gen_continuation_yield(masm,
2045                              in_regs,
2046                              oop_maps,
2047                              frame_complete,
2048                              stack_slots,
2049                              vep_offset);
2050     } else {
2051       guarantee(false, "Unknown Continuation native intrinsic");
2052     }
2053 
2054 #ifdef ASSERT
2055     if (method->is_continuation_enter_intrinsic()) {
2056       assert(interpreted_entry_offset != -1, "Must be set");
2057       assert(exception_offset != -1,         "Must be set");
2058     } else {
2059       assert(interpreted_entry_offset == -1, "Must be unset");
2060       assert(exception_offset == -1,         "Must be unset");
2061     }
2062     assert(frame_complete != -1,    "Must be set");
2063     assert(stack_slots != -1,       "Must be set");
2064     assert(vep_offset != -1,        "Must be set");
2065 #endif
2066 
2067     __ flush();
2068     nmethod* nm = nmethod::new_native_nmethod(method,
2069                                               compile_id,
2070                                               masm->code(),
2071                                               vep_offset,
2072                                               frame_complete,
2073                                               stack_slots,
2074                                               in_ByteSize(-1),
2075                                               in_ByteSize(-1),
2076                                               oop_maps,
2077                                               exception_offset);
2078     if (nm == nullptr) return nm;
2079     if (method->is_continuation_enter_intrinsic()) {
2080       ContinuationEntry::set_enter_code(nm, interpreted_entry_offset);
2081     } else if (method->is_continuation_yield_intrinsic()) {
2082       _cont_doYield_stub = nm;
2083     }
2084     return nm;
2085   }
2086 
2087   if (method->is_method_handle_intrinsic()) {
2088     vmIntrinsics::ID iid = method->intrinsic_id();
2089     intptr_t start = (intptr_t)__ pc();
2090     int vep_offset = ((intptr_t)__ pc()) - start;
2091     gen_special_dispatch(masm,
2092                          method,
2093                          in_sig_bt,
2094                          in_regs);
2095     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
2096     __ flush();
2097     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
2098     return nmethod::new_native_nmethod(method,
2099                                        compile_id,
2100                                        masm->code(),
2101                                        vep_offset,
2102                                        frame_complete,
2103                                        stack_slots / VMRegImpl::slots_per_word,
2104                                        in_ByteSize(-1),
2105                                        in_ByteSize(-1),
2106                                        nullptr);
2107   }
2108   address native_func = method->native_function();
2109   assert(native_func != nullptr, "must have function");
2110 
2111   // An OopMap for lock (and class if static)
2112   OopMapSet *oop_maps = new OopMapSet();
2113   intptr_t start = (intptr_t)__ pc();
2114 
2115   // We have received a description of where all the java arg are located
2116   // on entry to the wrapper. We need to convert these args to where
2117   // the jni function will expect them. To figure out where they go
2118   // we convert the java signature to a C signature by inserting
2119   // the hidden arguments as arg[0] and possibly arg[1] (static method)
2120 
2121   const int total_in_args = method->size_of_parameters();
2122   int total_c_args = total_in_args + (method->is_static() ? 2 : 1);
2123 
2124   BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
2125   VMRegPair* out_regs   = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
2126 
2127   int argc = 0;
2128   out_sig_bt[argc++] = T_ADDRESS;
2129   if (method->is_static()) {
2130     out_sig_bt[argc++] = T_OBJECT;
2131   }
2132 
2133   for (int i = 0; i < total_in_args ; i++ ) {
2134     out_sig_bt[argc++] = in_sig_bt[i];
2135   }
2136 
2137   // Now figure out where the args must be stored and how much stack space
2138   // they require.
2139   int out_arg_slots;
2140   out_arg_slots = c_calling_convention(out_sig_bt, out_regs, total_c_args);
2141 
2142   // Compute framesize for the wrapper.  We need to handlize all oops in
2143   // incoming registers
2144 
2145   // Calculate the total number of stack slots we will need.
2146 
2147   // First count the abi requirement plus all of the outgoing args
2148   int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots;
2149 
2150   // Now the space for the inbound oop handle area
2151   int total_save_slots = 6 * VMRegImpl::slots_per_word;  // 6 arguments passed in registers
2152 
2153   int oop_handle_offset = stack_slots;
2154   stack_slots += total_save_slots;
2155 
2156   // Now any space we need for handlizing a klass if static method
2157 
2158   int klass_slot_offset = 0;
2159   int klass_offset = -1;
2160   int lock_slot_offset = 0;
2161   bool is_static = false;
2162 
2163   if (method->is_static()) {
2164     klass_slot_offset = stack_slots;
2165     stack_slots += VMRegImpl::slots_per_word;
2166     klass_offset = klass_slot_offset * VMRegImpl::stack_slot_size;
2167     is_static = true;
2168   }
2169 
2170   // Plus a lock if needed
2171 
2172   if (method->is_synchronized()) {
2173     lock_slot_offset = stack_slots;
2174     stack_slots += VMRegImpl::slots_per_word;
2175   }
2176 
2177   // Now a place (+2) to save return values or temp during shuffling
2178   // + 4 for return address (which we own) and saved rbp
2179   stack_slots += 6;
2180 
2181   // Ok The space we have allocated will look like:
2182   //
2183   //
2184   // FP-> |                     |
2185   //      |---------------------|
2186   //      | 2 slots for moves   |
2187   //      |---------------------|
2188   //      | lock box (if sync)  |
2189   //      |---------------------| <- lock_slot_offset
2190   //      | klass (if static)   |
2191   //      |---------------------| <- klass_slot_offset
2192   //      | oopHandle area      |
2193   //      |---------------------| <- oop_handle_offset (6 java arg registers)
2194   //      | outbound memory     |
2195   //      | based arguments     |
2196   //      |                     |
2197   //      |---------------------|
2198   //      |                     |
2199   // SP-> | out_preserved_slots |
2200   //
2201   //
2202 
2203 
2204   // Now compute actual number of stack words we need rounding to make
2205   // stack properly aligned.
2206   stack_slots = align_up(stack_slots, StackAlignmentInSlots);
2207 
2208   int stack_size = stack_slots * VMRegImpl::stack_slot_size;
2209 
2210   // First thing make an ic check to see if we should even be here
2211 
2212   // We are free to use all registers as temps without saving them and
2213   // restoring them except rbp. rbp is the only callee save register
2214   // as far as the interpreter and the compiler(s) are concerned.
2215 
2216   const Register receiver = j_rarg0;
2217 
2218   Label exception_pending;
2219 
2220   assert_different_registers(receiver, rscratch1, rscratch2);
2221   __ verify_oop(receiver);
2222   __ ic_check(8 /* end_alignment */);
2223 
2224   int vep_offset = ((intptr_t)__ pc()) - start;
2225 
2226   if (VM_Version::supports_fast_class_init_checks() && method->needs_clinit_barrier()) {
2227     Label L_skip_barrier;
2228     Register klass = r10;
2229     __ mov_metadata(klass, method->method_holder()); // InstanceKlass*
2230     __ clinit_barrier(klass, &L_skip_barrier /*L_fast_path*/);
2231 
2232     __ jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); // slow path
2233 
2234     __ bind(L_skip_barrier);
2235   }
2236 
2237 #ifdef COMPILER1
2238   // For Object.hashCode, System.identityHashCode try to pull hashCode from object header if available.
2239   if ((InlineObjectHash && method->intrinsic_id() == vmIntrinsics::_hashCode) || (method->intrinsic_id() == vmIntrinsics::_identityHashCode)) {
2240     inline_check_hashcode_from_object_header(masm, method, j_rarg0 /*obj_reg*/, rax /*result*/);
2241   }
2242 #endif // COMPILER1
2243 
2244   // The instruction at the verified entry point must be 5 bytes or longer
2245   // because it can be patched on the fly by make_non_entrant. The stack bang
2246   // instruction fits that requirement.
2247 
2248   // Generate stack overflow check
2249   __ bang_stack_with_offset((int)StackOverflow::stack_shadow_zone_size());
2250 
2251   // Generate a new frame for the wrapper.
2252   __ enter();
2253   // -2 because return address is already present and so is saved rbp
2254   __ subptr(rsp, stack_size - 2*wordSize);
2255 
2256   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
2257   // native wrapper is not hot enough to micro optimize the nmethod entry barrier with an out-of-line stub
2258   bs->nmethod_entry_barrier(masm, nullptr /* slow_path */, nullptr /* continuation */);
2259 
2260   // Frame is now completed as far as size and linkage.
2261   int frame_complete = ((intptr_t)__ pc()) - start;
2262 
2263 #ifdef ASSERT
2264   __ check_stack_alignment(rsp, "improperly aligned stack");
2265 #endif /* ASSERT */
2266 
2267 
2268   // We use r14 as the oop handle for the receiver/klass
2269   // It is callee save so it survives the call to native
2270 
2271   const Register oop_handle_reg = r14;
2272 
2273   //
2274   // We immediately shuffle the arguments so that any vm call we have to
2275   // make from here on out (sync slow path, jvmti, etc.) we will have
2276   // captured the oops from our caller and have a valid oopMap for
2277   // them.
2278 
2279   // -----------------
2280   // The Grand Shuffle
2281 
2282   // The Java calling convention is either equal (linux) or denser (win64) than the
2283   // c calling convention. However the because of the jni_env argument the c calling
2284   // convention always has at least one more (and two for static) arguments than Java.
2285   // Therefore if we move the args from java -> c backwards then we will never have
2286   // a register->register conflict and we don't have to build a dependency graph
2287   // and figure out how to break any cycles.
2288   //
2289 
2290   // Record esp-based slot for receiver on stack for non-static methods
2291   int receiver_offset = -1;
2292 
2293   // This is a trick. We double the stack slots so we can claim
2294   // the oops in the caller's frame. Since we are sure to have
2295   // more args than the caller doubling is enough to make
2296   // sure we can capture all the incoming oop args from the
2297   // caller.
2298   //
2299   OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
2300 
2301   // Mark location of rbp (someday)
2302   // map->set_callee_saved(VMRegImpl::stack2reg( stack_slots - 2), stack_slots * 2, 0, vmreg(rbp));
2303 
2304   // Use eax, ebx as temporaries during any memory-memory moves we have to do
2305   // All inbound args are referenced based on rbp and all outbound args via rsp.
2306 
2307 
2308 #ifdef ASSERT
2309   bool reg_destroyed[Register::number_of_registers];
2310   bool freg_destroyed[XMMRegister::number_of_registers];
2311   for ( int r = 0 ; r < Register::number_of_registers ; r++ ) {
2312     reg_destroyed[r] = false;
2313   }
2314   for ( int f = 0 ; f < XMMRegister::number_of_registers ; f++ ) {
2315     freg_destroyed[f] = false;
2316   }
2317 
2318 #endif /* ASSERT */
2319 
2320   // For JNI natives the incoming and outgoing registers are offset upwards.
2321   GrowableArray<int> arg_order(2 * total_in_args);
2322 
2323   for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
2324     arg_order.push(i);
2325     arg_order.push(c_arg);
2326   }
2327 
2328   for (int ai = 0; ai < arg_order.length(); ai += 2) {
2329     int i = arg_order.at(ai);
2330     int c_arg = arg_order.at(ai + 1);
2331     __ block_comment(err_msg("move %d -> %d", i, c_arg));
2332 #ifdef ASSERT
2333     if (in_regs[i].first()->is_Register()) {
2334       assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
2335     } else if (in_regs[i].first()->is_XMMRegister()) {
2336       assert(!freg_destroyed[in_regs[i].first()->as_XMMRegister()->encoding()], "destroyed reg!");
2337     }
2338     if (out_regs[c_arg].first()->is_Register()) {
2339       reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
2340     } else if (out_regs[c_arg].first()->is_XMMRegister()) {
2341       freg_destroyed[out_regs[c_arg].first()->as_XMMRegister()->encoding()] = true;
2342     }
2343 #endif /* ASSERT */
2344     switch (in_sig_bt[i]) {
2345       case T_ARRAY:
2346       case T_OBJECT:
2347         __ object_move(map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
2348                     ((i == 0) && (!is_static)),
2349                     &receiver_offset);
2350         break;
2351       case T_VOID:
2352         break;
2353 
2354       case T_FLOAT:
2355         __ float_move(in_regs[i], out_regs[c_arg]);
2356           break;
2357 
2358       case T_DOUBLE:
2359         assert( i + 1 < total_in_args &&
2360                 in_sig_bt[i + 1] == T_VOID &&
2361                 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
2362         __ double_move(in_regs[i], out_regs[c_arg]);
2363         break;
2364 
2365       case T_LONG :
2366         __ long_move(in_regs[i], out_regs[c_arg]);
2367         break;
2368 
2369       case T_ADDRESS: assert(false, "found T_ADDRESS in java args");
2370 
2371       default:
2372         __ move32_64(in_regs[i], out_regs[c_arg]);
2373     }
2374   }
2375 
2376   int c_arg;
2377 
2378   // Pre-load a static method's oop into r14.  Used both by locking code and
2379   // the normal JNI call code.
2380   // point c_arg at the first arg that is already loaded in case we
2381   // need to spill before we call out
2382   c_arg = total_c_args - total_in_args;
2383 
2384   if (method->is_static()) {
2385 
2386     //  load oop into a register
2387     __ movoop(oop_handle_reg, JNIHandles::make_local(method->method_holder()->java_mirror()));
2388 
2389     // Now handlize the static class mirror it's known not-null.
2390     __ movptr(Address(rsp, klass_offset), oop_handle_reg);
2391     map->set_oop(VMRegImpl::stack2reg(klass_slot_offset));
2392 
2393     // Now get the handle
2394     __ lea(oop_handle_reg, Address(rsp, klass_offset));
2395     // store the klass handle as second argument
2396     __ movptr(c_rarg1, oop_handle_reg);
2397     // and protect the arg if we must spill
2398     c_arg--;
2399   }
2400 
2401   // Change state to native (we save the return address in the thread, since it might not
2402   // be pushed on the stack when we do a stack traversal). It is enough that the pc()
2403   // points into the right code segment. It does not have to be the correct return pc.
2404   // We use the same pc/oopMap repeatedly when we call out
2405 
2406   Label native_return;
2407   if (method->is_object_wait0()) {
2408     // For convenience we use the pc we want to resume to in case of preemption on Object.wait.
2409     __ set_last_Java_frame(rsp, noreg, native_return, rscratch1);
2410   } else {
2411     intptr_t the_pc = (intptr_t) __ pc();
2412     oop_maps->add_gc_map(the_pc - start, map);
2413 
2414     __ set_last_Java_frame(rsp, noreg, __ pc(), rscratch1);
2415   }
2416 
2417   // We have all of the arguments setup at this point. We must not touch any register
2418   // argument registers at this point (what if we save/restore them there are no oop?
2419 
2420   if (DTraceMethodProbes) {
2421     // protect the args we've loaded
2422     save_args(masm, total_c_args, c_arg, out_regs);
2423     __ mov_metadata(c_rarg1, method());
2424     __ call_VM_leaf(
2425       CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
2426       r15_thread, c_rarg1);
2427     restore_args(masm, total_c_args, c_arg, out_regs);
2428   }
2429 
2430   // RedefineClasses() tracing support for obsolete method entry
2431   if (log_is_enabled(Trace, redefine, class, obsolete)) {
2432     // protect the args we've loaded
2433     save_args(masm, total_c_args, c_arg, out_regs);
2434     __ mov_metadata(c_rarg1, method());
2435     __ call_VM_leaf(
2436       CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
2437       r15_thread, c_rarg1);
2438     restore_args(masm, total_c_args, c_arg, out_regs);
2439   }
2440 
2441   // Lock a synchronized method
2442 
2443   // Register definitions used by locking and unlocking
2444 
2445   const Register swap_reg = rax;  // Must use rax for cmpxchg instruction
2446   const Register obj_reg  = rbx;  // Will contain the oop
2447   const Register lock_reg = r13;  // Address of compiler lock object (BasicLock)
2448 
2449   Label slow_path_lock;
2450   Label lock_done;
2451 
2452   if (method->is_synchronized()) {
2453     // Get the handle (the 2nd argument)
2454     __ mov(oop_handle_reg, c_rarg1);
2455 
2456     // Get address of the box
2457 
2458     __ lea(lock_reg, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2459 
2460     // Load the oop from the handle
2461     __ movptr(obj_reg, Address(oop_handle_reg, 0));
2462 
2463     __ lightweight_lock(lock_reg, obj_reg, swap_reg, rscratch1, slow_path_lock);
2464 
2465     // Slow path will re-enter here
2466     __ bind(lock_done);
2467   }
2468 
2469   // Finally just about ready to make the JNI call
2470 
2471   // get JNIEnv* which is first argument to native
2472   __ lea(c_rarg0, Address(r15_thread, in_bytes(JavaThread::jni_environment_offset())));
2473 
2474   // Now set thread in native
2475   __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native);
2476 
2477   __ call(RuntimeAddress(native_func));
2478 
2479   // Verify or restore cpu control state after JNI call
2480   __ restore_cpu_control_state_after_jni(rscratch1);
2481 
2482   // Unpack native results.
2483   switch (ret_type) {
2484   case T_BOOLEAN: __ c2bool(rax);            break;
2485   case T_CHAR   : __ movzwl(rax, rax);      break;
2486   case T_BYTE   : __ sign_extend_byte (rax); break;
2487   case T_SHORT  : __ sign_extend_short(rax); break;
2488   case T_INT    : /* nothing to do */        break;
2489   case T_DOUBLE :
2490   case T_FLOAT  :
2491     // Result is in xmm0 we'll save as needed
2492     break;
2493   case T_ARRAY:                 // Really a handle
2494   case T_OBJECT:                // Really a handle
2495       break; // can't de-handlize until after safepoint check
2496   case T_VOID: break;
2497   case T_LONG: break;
2498   default       : ShouldNotReachHere();
2499   }
2500 
2501   // Switch thread to "native transition" state before reading the synchronization state.
2502   // This additional state is necessary because reading and testing the synchronization
2503   // state is not atomic w.r.t. GC, as this scenario demonstrates:
2504   //     Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2505   //     VM thread changes sync state to synchronizing and suspends threads for GC.
2506   //     Thread A is resumed to finish this native method, but doesn't block here since it
2507   //     didn't see any synchronization is progress, and escapes.
2508   __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_native_trans);
2509 
2510   // Force this write out before the read below
2511   if (!UseSystemMemoryBarrier) {
2512     __ membar(Assembler::Membar_mask_bits(
2513               Assembler::LoadLoad | Assembler::LoadStore |
2514               Assembler::StoreLoad | Assembler::StoreStore));
2515   }
2516 
2517   // check for safepoint operation in progress and/or pending suspend requests
2518   {
2519     Label Continue;
2520     Label slow_path;
2521 
2522     __ safepoint_poll(slow_path, true /* at_return */, false /* in_nmethod */);
2523 
2524     __ cmpl(Address(r15_thread, JavaThread::suspend_flags_offset()), 0);
2525     __ jcc(Assembler::equal, Continue);
2526     __ bind(slow_path);
2527 
2528     // Don't use call_VM as it will see a possible pending exception and forward it
2529     // and never return here preventing us from clearing _last_native_pc down below.
2530     // Also can't use call_VM_leaf either as it will check to see if rsi & rdi are
2531     // preserved and correspond to the bcp/locals pointers. So we do a runtime call
2532     // by hand.
2533     //
2534     __ vzeroupper();
2535     save_native_result(masm, ret_type, stack_slots);
2536     __ mov(c_rarg0, r15_thread);
2537     __ mov(r12, rsp); // remember sp
2538     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
2539     __ andptr(rsp, -16); // align stack as required by ABI
2540     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
2541     __ mov(rsp, r12); // restore sp
2542     __ reinit_heapbase();
2543     // Restore any method result value
2544     restore_native_result(masm, ret_type, stack_slots);
2545     __ bind(Continue);
2546   }
2547 
2548   // change thread state
2549   __ movl(Address(r15_thread, JavaThread::thread_state_offset()), _thread_in_Java);
2550 
2551   if (method->is_object_wait0()) {
2552     // Check preemption for Object.wait()
2553     __ movptr(rscratch1, Address(r15_thread, JavaThread::preempt_alternate_return_offset()));
2554     __ cmpptr(rscratch1, NULL_WORD);
2555     __ jccb(Assembler::equal, native_return);
2556     __ movptr(Address(r15_thread, JavaThread::preempt_alternate_return_offset()), NULL_WORD);
2557     __ jmp(rscratch1);
2558     __ bind(native_return);
2559 
2560     intptr_t the_pc = (intptr_t) __ pc();
2561     oop_maps->add_gc_map(the_pc - start, map);
2562   }
2563 
2564 
2565   Label reguard;
2566   Label reguard_done;
2567   __ cmpl(Address(r15_thread, JavaThread::stack_guard_state_offset()), StackOverflow::stack_guard_yellow_reserved_disabled);
2568   __ jcc(Assembler::equal, reguard);
2569   __ bind(reguard_done);
2570 
2571   // native result if any is live
2572 
2573   // Unlock
2574   Label slow_path_unlock;
2575   Label unlock_done;
2576   if (method->is_synchronized()) {
2577 
2578     Label fast_done;
2579 
2580     // Get locked oop from the handle we passed to jni
2581     __ movptr(obj_reg, Address(oop_handle_reg, 0));
2582 
2583     // Must save rax if it is live now because cmpxchg must use it
2584     if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2585       save_native_result(masm, ret_type, stack_slots);
2586     }
2587 
2588     __ lightweight_unlock(obj_reg, swap_reg, lock_reg, slow_path_unlock);
2589 
2590     // slow path re-enters here
2591     __ bind(unlock_done);
2592     if (ret_type != T_FLOAT && ret_type != T_DOUBLE && ret_type != T_VOID) {
2593       restore_native_result(masm, ret_type, stack_slots);
2594     }
2595 
2596     __ bind(fast_done);
2597   }
2598   if (DTraceMethodProbes) {
2599     save_native_result(masm, ret_type, stack_slots);
2600     __ mov_metadata(c_rarg1, method());
2601     __ call_VM_leaf(
2602          CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2603          r15_thread, c_rarg1);
2604     restore_native_result(masm, ret_type, stack_slots);
2605   }
2606 
2607   __ reset_last_Java_frame(false);
2608 
2609   // Unbox oop result, e.g. JNIHandles::resolve value.
2610   if (is_reference_type(ret_type)) {
2611     __ resolve_jobject(rax /* value */,
2612                        rcx /* tmp */);
2613   }
2614 
2615   if (CheckJNICalls) {
2616     // clear_pending_jni_exception_check
2617     __ movptr(Address(r15_thread, JavaThread::pending_jni_exception_check_fn_offset()), NULL_WORD);
2618   }
2619 
2620   // reset handle block
2621   __ movptr(rcx, Address(r15_thread, JavaThread::active_handles_offset()));
2622   __ movl(Address(rcx, JNIHandleBlock::top_offset()), NULL_WORD);
2623 
2624   // pop our frame
2625 
2626   __ leave();
2627 
2628 #if INCLUDE_JFR
2629   // We need to do a poll test after unwind in case the sampler
2630   // managed to sample the native frame after returning to Java.
2631   Label L_return;
2632   address poll_test_pc = __ pc();
2633   __ relocate(relocInfo::poll_return_type);
2634   __ testb(Address(r15_thread, JavaThread::polling_word_offset()), SafepointMechanism::poll_bit());
2635   __ jccb(Assembler::zero, L_return);
2636   __ lea(rscratch1, InternalAddress(poll_test_pc));
2637   __ movptr(Address(r15_thread, JavaThread::saved_exception_pc_offset()), rscratch1);
2638   assert(SharedRuntime::polling_page_return_handler_blob() != nullptr,
2639     "polling page return stub not created yet");
2640   address stub = SharedRuntime::polling_page_return_handler_blob()->entry_point();
2641   __ jump(RuntimeAddress(stub));
2642   __ bind(L_return);
2643 #endif // INCLUDE_JFR
2644 
2645   // Any exception pending?
2646   __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), NULL_WORD);
2647   __ jcc(Assembler::notEqual, exception_pending);
2648 
2649   // Return
2650 
2651   __ ret(0);
2652 
2653   // Unexpected paths are out of line and go here
2654 
2655   // forward the exception
2656   __ bind(exception_pending);
2657 
2658   // and forward the exception
2659   __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2660 
2661   // Slow path locking & unlocking
2662   if (method->is_synchronized()) {
2663 
2664     // BEGIN Slow path lock
2665     __ bind(slow_path_lock);
2666 
2667     // has last_Java_frame setup. No exceptions so do vanilla call not call_VM
2668     // args are (oop obj, BasicLock* lock, JavaThread* thread)
2669 
2670     // protect the args we've loaded
2671     save_args(masm, total_c_args, c_arg, out_regs);
2672 
2673     __ mov(c_rarg0, obj_reg);
2674     __ mov(c_rarg1, lock_reg);
2675     __ mov(c_rarg2, r15_thread);
2676 
2677     // Not a leaf but we have last_Java_frame setup as we want.
2678     // We don't want to unmount in case of contention since that would complicate preserving
2679     // the arguments that had already been marshalled into the native convention. So we force
2680     // the freeze slow path to find this native wrapper frame (see recurse_freeze_native_frame())
2681     // and pin the vthread. Otherwise the fast path won't find it since we don't walk the stack.
2682     __ push_cont_fastpath();
2683     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3);
2684     __ pop_cont_fastpath();
2685     restore_args(masm, total_c_args, c_arg, out_regs);
2686 
2687 #ifdef ASSERT
2688     { Label L;
2689     __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), NULL_WORD);
2690     __ jcc(Assembler::equal, L);
2691     __ stop("no pending exception allowed on exit from monitorenter");
2692     __ bind(L);
2693     }
2694 #endif
2695     __ jmp(lock_done);
2696 
2697     // END Slow path lock
2698 
2699     // BEGIN Slow path unlock
2700     __ bind(slow_path_unlock);
2701 
2702     // If we haven't already saved the native result we must save it now as xmm registers
2703     // are still exposed.
2704     __ vzeroupper();
2705     if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
2706       save_native_result(masm, ret_type, stack_slots);
2707     }
2708 
2709     __ lea(c_rarg1, Address(rsp, lock_slot_offset * VMRegImpl::stack_slot_size));
2710 
2711     __ mov(c_rarg0, obj_reg);
2712     __ mov(c_rarg2, r15_thread);
2713     __ mov(r12, rsp); // remember sp
2714     __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
2715     __ andptr(rsp, -16); // align stack as required by ABI
2716 
2717     // Save pending exception around call to VM (which contains an EXCEPTION_MARK)
2718     // NOTE that obj_reg == rbx currently
2719     __ movptr(rbx, Address(r15_thread, in_bytes(Thread::pending_exception_offset())));
2720     __ movptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), NULL_WORD);
2721 
2722     // args are (oop obj, BasicLock* lock, JavaThread* thread)
2723     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C)));
2724     __ mov(rsp, r12); // restore sp
2725     __ reinit_heapbase();
2726 #ifdef ASSERT
2727     {
2728       Label L;
2729       __ cmpptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), NULL_WORD);
2730       __ jcc(Assembler::equal, L);
2731       __ stop("no pending exception allowed on exit complete_monitor_unlocking_C");
2732       __ bind(L);
2733     }
2734 #endif /* ASSERT */
2735 
2736     __ movptr(Address(r15_thread, in_bytes(Thread::pending_exception_offset())), rbx);
2737 
2738     if (ret_type == T_FLOAT || ret_type == T_DOUBLE ) {
2739       restore_native_result(masm, ret_type, stack_slots);
2740     }
2741     __ jmp(unlock_done);
2742 
2743     // END Slow path unlock
2744 
2745   } // synchronized
2746 
2747   // SLOW PATH Reguard the stack if needed
2748 
2749   __ bind(reguard);
2750   __ vzeroupper();
2751   save_native_result(masm, ret_type, stack_slots);
2752   __ mov(r12, rsp); // remember sp
2753   __ subptr(rsp, frame::arg_reg_save_area_bytes); // windows
2754   __ andptr(rsp, -16); // align stack as required by ABI
2755   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages)));
2756   __ mov(rsp, r12); // restore sp
2757   __ reinit_heapbase();
2758   restore_native_result(masm, ret_type, stack_slots);
2759   // and continue
2760   __ jmp(reguard_done);
2761 
2762 
2763 
2764   __ flush();
2765 
2766   nmethod *nm = nmethod::new_native_nmethod(method,
2767                                             compile_id,
2768                                             masm->code(),
2769                                             vep_offset,
2770                                             frame_complete,
2771                                             stack_slots / VMRegImpl::slots_per_word,
2772                                             (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)),
2773                                             in_ByteSize(lock_slot_offset*VMRegImpl::stack_slot_size),
2774                                             oop_maps);
2775 
2776   return nm;
2777 }
2778 
2779 // this function returns the adjust size (in number of words) to a c2i adapter
2780 // activation for use during deoptimization
2781 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals ) {
2782   return (callee_locals - callee_parameters) * Interpreter::stackElementWords;
2783 }
2784 
2785 
2786 uint SharedRuntime::out_preserve_stack_slots() {
2787   return 0;
2788 }
2789 
2790 
2791 // Number of stack slots between incoming argument block and the start of
2792 // a new frame.  The PROLOG must add this many slots to the stack.  The
2793 // EPILOG must remove this many slots.  amd64 needs two slots for
2794 // return address.
2795 uint SharedRuntime::in_preserve_stack_slots() {
2796   return 4 + 2 * VerifyStackAtCalls;
2797 }
2798 
2799 VMReg SharedRuntime::thread_register() {
2800   return r15_thread->as_VMReg();
2801 }
2802 
2803 //------------------------------generate_deopt_blob----------------------------
2804 void SharedRuntime::generate_deopt_blob() {
2805   // Allocate space for the code
2806   ResourceMark rm;
2807   // Setup code generation tools
2808   int pad = 0;
2809   if (UseAVX > 2) {
2810     pad += 1024;
2811   }
2812   if (UseAPX) {
2813     pad += 1024;
2814   }
2815 #if INCLUDE_JVMCI
2816   if (EnableJVMCI) {
2817     pad += 512; // Increase the buffer size when compiling for JVMCI
2818   }
2819 #endif
2820   const char* name = SharedRuntime::stub_name(StubId::shared_deopt_id);
2821   CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::SharedBlob, BlobId::shared_deopt_id);
2822   if (blob != nullptr) {
2823     _deopt_blob = blob->as_deoptimization_blob();
2824     return;
2825   }
2826 
2827   CodeBuffer buffer(name, 2560+pad, 1024);
2828   MacroAssembler* masm = new MacroAssembler(&buffer);
2829   int frame_size_in_words;
2830   OopMap* map = nullptr;
2831   OopMapSet *oop_maps = new OopMapSet();
2832 
2833   // -------------
2834   // This code enters when returning to a de-optimized nmethod.  A return
2835   // address has been pushed on the stack, and return values are in
2836   // registers.
2837   // If we are doing a normal deopt then we were called from the patched
2838   // nmethod from the point we returned to the nmethod. So the return
2839   // address on the stack is wrong by NativeCall::instruction_size
2840   // We will adjust the value so it looks like we have the original return
2841   // address on the stack (like when we eagerly deoptimized).
2842   // In the case of an exception pending when deoptimizing, we enter
2843   // with a return address on the stack that points after the call we patched
2844   // into the exception handler. We have the following register state from,
2845   // e.g., the forward exception stub (see stubGenerator_x86_64.cpp).
2846   //    rax: exception oop
2847   //    rbx: exception handler
2848   //    rdx: throwing pc
2849   // So in this case we simply jam rdx into the useless return address and
2850   // the stack looks just like we want.
2851   //
2852   // At this point we need to de-opt.  We save the argument return
2853   // registers.  We call the first C routine, fetch_unroll_info().  This
2854   // routine captures the return values and returns a structure which
2855   // describes the current frame size and the sizes of all replacement frames.
2856   // The current frame is compiled code and may contain many inlined
2857   // functions, each with their own JVM state.  We pop the current frame, then
2858   // push all the new frames.  Then we call the C routine unpack_frames() to
2859   // populate these frames.  Finally unpack_frames() returns us the new target
2860   // address.  Notice that callee-save registers are BLOWN here; they have
2861   // already been captured in the vframeArray at the time the return PC was
2862   // patched.
2863   address start = __ pc();
2864   Label cont;
2865 
2866   // Prolog for non exception case!
2867 
2868   // Save everything in sight.
2869   map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_wide_vectors*/ true);
2870 
2871   // Normal deoptimization.  Save exec mode for unpack_frames.
2872   __ movl(r14, Deoptimization::Unpack_deopt); // callee-saved
2873   __ jmp(cont);
2874 
2875   int reexecute_offset = __ pc() - start;
2876 #if INCLUDE_JVMCI && !defined(COMPILER1)
2877   if (UseJVMCICompiler) {
2878     // JVMCI does not use this kind of deoptimization
2879     __ should_not_reach_here();
2880   }
2881 #endif
2882 
2883   // Reexecute case
2884   // return address is the pc describes what bci to do re-execute at
2885 
2886   // No need to update map as each call to save_live_registers will produce identical oopmap
2887   (void) RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_wide_vectors*/ true);
2888 
2889   __ movl(r14, Deoptimization::Unpack_reexecute); // callee-saved
2890   __ jmp(cont);
2891 
2892 #if INCLUDE_JVMCI
2893   Label after_fetch_unroll_info_call;
2894   int implicit_exception_uncommon_trap_offset = 0;
2895   int uncommon_trap_offset = 0;
2896 
2897   if (EnableJVMCI) {
2898     implicit_exception_uncommon_trap_offset = __ pc() - start;
2899 
2900     __ pushptr(Address(r15_thread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())));
2901     __ movptr(Address(r15_thread, in_bytes(JavaThread::jvmci_implicit_exception_pc_offset())), NULL_WORD);
2902 
2903     uncommon_trap_offset = __ pc() - start;
2904 
2905     // Save everything in sight.
2906     RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_wide_vectors*/ true);
2907     // fetch_unroll_info needs to call last_java_frame()
2908     __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1);
2909 
2910     __ movl(c_rarg1, Address(r15_thread, in_bytes(JavaThread::pending_deoptimization_offset())));
2911     __ movl(Address(r15_thread, in_bytes(JavaThread::pending_deoptimization_offset())), -1);
2912 
2913     __ movl(r14, Deoptimization::Unpack_reexecute);
2914     __ mov(c_rarg0, r15_thread);
2915     __ movl(c_rarg2, r14); // exec mode
2916     __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::uncommon_trap)));
2917     oop_maps->add_gc_map( __ pc()-start, map->deep_copy());
2918 
2919     __ reset_last_Java_frame(false);
2920 
2921     __ jmp(after_fetch_unroll_info_call);
2922   } // EnableJVMCI
2923 #endif // INCLUDE_JVMCI
2924 
2925   int exception_offset = __ pc() - start;
2926 
2927   // Prolog for exception case
2928 
2929   // all registers are dead at this entry point, except for rax, and
2930   // rdx which contain the exception oop and exception pc
2931   // respectively.  Set them in TLS and fall thru to the
2932   // unpack_with_exception_in_tls entry point.
2933 
2934   __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), rdx);
2935   __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), rax);
2936 
2937   int exception_in_tls_offset = __ pc() - start;
2938 
2939   // new implementation because exception oop is now passed in JavaThread
2940 
2941   // Prolog for exception case
2942   // All registers must be preserved because they might be used by LinearScan
2943   // Exceptiop oop and throwing PC are passed in JavaThread
2944   // tos: stack at point of call to method that threw the exception (i.e. only
2945   // args are on the stack, no return address)
2946 
2947   // make room on stack for the return address
2948   // It will be patched later with the throwing pc. The correct value is not
2949   // available now because loading it from memory would destroy registers.
2950   __ push(0);
2951 
2952   // Save everything in sight.
2953   map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_wide_vectors*/ true);
2954 
2955   // Now it is safe to overwrite any register
2956 
2957   // Deopt during an exception.  Save exec mode for unpack_frames.
2958   __ movl(r14, Deoptimization::Unpack_exception); // callee-saved
2959 
2960   // load throwing pc from JavaThread and patch it as the return address
2961   // of the current frame. Then clear the field in JavaThread
2962 
2963   __ movptr(rdx, Address(r15_thread, JavaThread::exception_pc_offset()));
2964   __ movptr(Address(rbp, wordSize), rdx);
2965   __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
2966 
2967 #ifdef ASSERT
2968   // verify that there is really an exception oop in JavaThread
2969   __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
2970   __ verify_oop(rax);
2971 
2972   // verify that there is no pending exception
2973   Label no_pending_exception;
2974   __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
2975   __ testptr(rax, rax);
2976   __ jcc(Assembler::zero, no_pending_exception);
2977   __ stop("must not have pending exception here");
2978   __ bind(no_pending_exception);
2979 #endif
2980 
2981   __ bind(cont);
2982 
2983   // Call C code.  Need thread and this frame, but NOT official VM entry
2984   // crud.  We cannot block on this call, no GC can happen.
2985   //
2986   // UnrollBlock* fetch_unroll_info(JavaThread* thread)
2987 
2988   // fetch_unroll_info needs to call last_java_frame().
2989 
2990   __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1);
2991 #ifdef ASSERT
2992   { Label L;
2993     __ cmpptr(Address(r15_thread, JavaThread::last_Java_fp_offset()), NULL_WORD);
2994     __ jcc(Assembler::equal, L);
2995     __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared");
2996     __ bind(L);
2997   }
2998 #endif // ASSERT
2999   __ mov(c_rarg0, r15_thread);
3000   __ movl(c_rarg1, r14); // exec_mode
3001   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info)));
3002 
3003   // Need to have an oopmap that tells fetch_unroll_info where to
3004   // find any register it might need.
3005   oop_maps->add_gc_map(__ pc() - start, map);
3006 
3007   __ reset_last_Java_frame(false);
3008 
3009 #if INCLUDE_JVMCI
3010   if (EnableJVMCI) {
3011     __ bind(after_fetch_unroll_info_call);
3012   }
3013 #endif
3014 
3015   // Load UnrollBlock* into rdi
3016   __ mov(rdi, rax);
3017 
3018   __ movl(r14, Address(rdi, Deoptimization::UnrollBlock::unpack_kind_offset()));
3019    Label noException;
3020   __ cmpl(r14, Deoptimization::Unpack_exception);   // Was exception pending?
3021   __ jcc(Assembler::notEqual, noException);
3022   __ movptr(rax, Address(r15_thread, JavaThread::exception_oop_offset()));
3023   // QQQ this is useless it was null above
3024   __ movptr(rdx, Address(r15_thread, JavaThread::exception_pc_offset()));
3025   __ movptr(Address(r15_thread, JavaThread::exception_oop_offset()), NULL_WORD);
3026   __ movptr(Address(r15_thread, JavaThread::exception_pc_offset()), NULL_WORD);
3027 
3028   __ verify_oop(rax);
3029 
3030   // Overwrite the result registers with the exception results.
3031   __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax);
3032   // I think this is useless
3033   __ movptr(Address(rsp, RegisterSaver::rdx_offset_in_bytes()), rdx);
3034 
3035   __ bind(noException);
3036 
3037   // Only register save data is on the stack.
3038   // Now restore the result registers.  Everything else is either dead
3039   // or captured in the vframeArray.
3040   RegisterSaver::restore_result_registers(masm);
3041 
3042   // All of the register save area has been popped of the stack. Only the
3043   // return address remains.
3044 
3045   // Pop all the frames we must move/replace.
3046   //
3047   // Frame picture (youngest to oldest)
3048   // 1: self-frame (no frame link)
3049   // 2: deopting frame  (no frame link)
3050   // 3: caller of deopting frame (could be compiled/interpreted).
3051   //
3052   // Note: by leaving the return address of self-frame on the stack
3053   // and using the size of frame 2 to adjust the stack
3054   // when we are done the return to frame 3 will still be on the stack.
3055 
3056   // Pop deoptimized frame
3057   __ movl(rcx, Address(rdi, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset()));
3058   __ addptr(rsp, rcx);
3059 
3060   // rsp should be pointing at the return address to the caller (3)
3061 
3062   // Pick up the initial fp we should save
3063   // restore rbp before stack bang because if stack overflow is thrown it needs to be pushed (and preserved)
3064   __ movptr(rbp, Address(rdi, Deoptimization::UnrollBlock::initial_info_offset()));
3065 
3066 #ifdef ASSERT
3067   // Compilers generate code that bang the stack by as much as the
3068   // interpreter would need. So this stack banging should never
3069   // trigger a fault. Verify that it does not on non product builds.
3070   __ movl(rbx, Address(rdi, Deoptimization::UnrollBlock::total_frame_sizes_offset()));
3071   __ bang_stack_size(rbx, rcx);
3072 #endif
3073 
3074   // Load address of array of frame pcs into rcx
3075   __ movptr(rcx, Address(rdi, Deoptimization::UnrollBlock::frame_pcs_offset()));
3076 
3077   // Trash the old pc
3078   __ addptr(rsp, wordSize);
3079 
3080   // Load address of array of frame sizes into rsi
3081   __ movptr(rsi, Address(rdi, Deoptimization::UnrollBlock::frame_sizes_offset()));
3082 
3083   // Load counter into rdx
3084   __ movl(rdx, Address(rdi, Deoptimization::UnrollBlock::number_of_frames_offset()));
3085 
3086   // Now adjust the caller's stack to make up for the extra locals
3087   // but record the original sp so that we can save it in the skeletal interpreter
3088   // frame and the stack walking of interpreter_sender will get the unextended sp
3089   // value and not the "real" sp value.
3090 
3091   const Register sender_sp = r8;
3092 
3093   __ mov(sender_sp, rsp);
3094   __ movl(rbx, Address(rdi,
3095                        Deoptimization::UnrollBlock::
3096                        caller_adjustment_offset()));
3097   __ subptr(rsp, rbx);
3098 
3099   // Push interpreter frames in a loop
3100   Label loop;
3101   __ bind(loop);
3102   __ movptr(rbx, Address(rsi, 0));      // Load frame size
3103   __ subptr(rbx, 2*wordSize);           // We'll push pc and ebp by hand
3104   __ pushptr(Address(rcx, 0));          // Save return address
3105   __ enter();                           // Save old & set new ebp
3106   __ subptr(rsp, rbx);                  // Prolog
3107   // This value is corrected by layout_activation_impl
3108   __ movptr(Address(rbp, frame::interpreter_frame_last_sp_offset * wordSize), NULL_WORD);
3109   __ movptr(Address(rbp, frame::interpreter_frame_sender_sp_offset * wordSize), sender_sp); // Make it walkable
3110   __ mov(sender_sp, rsp);               // Pass sender_sp to next frame
3111   __ addptr(rsi, wordSize);             // Bump array pointer (sizes)
3112   __ addptr(rcx, wordSize);             // Bump array pointer (pcs)
3113   __ decrementl(rdx);                   // Decrement counter
3114   __ jcc(Assembler::notZero, loop);
3115   __ pushptr(Address(rcx, 0));          // Save final return address
3116 
3117   // Re-push self-frame
3118   __ enter();                           // Save old & set new ebp
3119 
3120   // Allocate a full sized register save area.
3121   // Return address and rbp are in place, so we allocate two less words.
3122   __ subptr(rsp, (frame_size_in_words - 2) * wordSize);
3123 
3124   // Restore frame locals after moving the frame
3125   __ movdbl(Address(rsp, RegisterSaver::xmm0_offset_in_bytes()), xmm0);
3126   __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax);
3127 
3128   // Call C code.  Need thread but NOT official VM entry
3129   // crud.  We cannot block on this call, no GC can happen.  Call should
3130   // restore return values to their stack-slots with the new SP.
3131   //
3132   // void Deoptimization::unpack_frames(JavaThread* thread, int exec_mode)
3133 
3134   // Use rbp because the frames look interpreted now
3135   // Save "the_pc" since it cannot easily be retrieved using the last_java_SP after we aligned SP.
3136   // Don't need the precise return PC here, just precise enough to point into this code blob.
3137   address the_pc = __ pc();
3138   __ set_last_Java_frame(noreg, rbp, the_pc, rscratch1);
3139 
3140   __ andptr(rsp, -(StackAlignmentInBytes));  // Fix stack alignment as required by ABI
3141   __ mov(c_rarg0, r15_thread);
3142   __ movl(c_rarg1, r14); // second arg: exec_mode
3143   __ call(RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
3144   // Revert SP alignment after call since we're going to do some SP relative addressing below
3145   __ movptr(rsp, Address(r15_thread, JavaThread::last_Java_sp_offset()));
3146 
3147   // Set an oopmap for the call site
3148   // Use the same PC we used for the last java frame
3149   oop_maps->add_gc_map(the_pc - start,
3150                        new OopMap( frame_size_in_words, 0 ));
3151 
3152   // Clear fp AND pc
3153   __ reset_last_Java_frame(true);
3154 
3155   // Collect return values
3156   __ movdbl(xmm0, Address(rsp, RegisterSaver::xmm0_offset_in_bytes()));
3157   __ movptr(rax, Address(rsp, RegisterSaver::rax_offset_in_bytes()));
3158   // I think this is useless (throwing pc?)
3159   __ movptr(rdx, Address(rsp, RegisterSaver::rdx_offset_in_bytes()));
3160 
3161   // Pop self-frame.
3162   __ leave();                           // Epilog
3163 
3164   // Jump to interpreter
3165   __ ret(0);
3166 
3167   // Make sure all code is generated
3168   masm->flush();
3169 
3170   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
3171   _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
3172 #if INCLUDE_JVMCI
3173   if (EnableJVMCI) {
3174     _deopt_blob->set_uncommon_trap_offset(uncommon_trap_offset);
3175     _deopt_blob->set_implicit_exception_uncommon_trap_offset(implicit_exception_uncommon_trap_offset);
3176   }
3177 #endif
3178 
3179   AOTCodeCache::store_code_blob(*_deopt_blob, AOTCodeEntry::SharedBlob, BlobId::shared_deopt_id);
3180 }
3181 
3182 //------------------------------generate_handler_blob------
3183 //
3184 // Generate a special Compile2Runtime blob that saves all registers,
3185 // and setup oopmap.
3186 //
3187 SafepointBlob* SharedRuntime::generate_handler_blob(StubId id, address call_ptr) {
3188   assert(StubRoutines::forward_exception_entry() != nullptr,
3189          "must be generated before");
3190   assert(is_polling_page_id(id), "expected a polling page stub id");
3191 
3192   // Allocate space for the code.  Setup code generation tools.
3193   const char* name = SharedRuntime::stub_name(id);
3194   CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::SharedBlob, StubInfo::blob(id));
3195   if (blob != nullptr) {
3196     return blob->as_safepoint_blob();
3197   }
3198 
3199   ResourceMark rm;
3200   OopMapSet *oop_maps = new OopMapSet();
3201   OopMap* map;
3202   CodeBuffer buffer(name, 2548, 1024);
3203   MacroAssembler* masm = new MacroAssembler(&buffer);
3204 
3205   address start   = __ pc();
3206   address call_pc = nullptr;
3207   int frame_size_in_words;
3208   bool cause_return = (id == StubId::shared_polling_page_return_handler_id);
3209   bool save_wide_vectors = (id == StubId::shared_polling_page_vectors_safepoint_handler_id);
3210 
3211   // Make room for return address (or push it again)
3212   if (!cause_return) {
3213     __ push(rbx);
3214   }
3215 
3216   // Save registers, fpu state, and flags
3217   map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, save_wide_vectors);
3218 
3219   // The following is basically a call_VM.  However, we need the precise
3220   // address of the call in order to generate an oopmap. Hence, we do all the
3221   // work ourselves.
3222 
3223   __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1);  // JavaFrameAnchor::capture_last_Java_pc() will get the pc from the return address, which we store next:
3224 
3225   // The return address must always be correct so that frame constructor never
3226   // sees an invalid pc.
3227 
3228   if (!cause_return) {
3229     // Get the return pc saved by the signal handler and stash it in its appropriate place on the stack.
3230     // Additionally, rbx is a callee saved register and we can look at it later to determine
3231     // if someone changed the return address for us!
3232     __ movptr(rbx, Address(r15_thread, JavaThread::saved_exception_pc_offset()));
3233     __ movptr(Address(rbp, wordSize), rbx);
3234   }
3235 
3236   // Do the call
3237   __ mov(c_rarg0, r15_thread);
3238   __ call(RuntimeAddress(call_ptr));
3239 
3240   // Set an oopmap for the call site.  This oopmap will map all
3241   // oop-registers and debug-info registers as callee-saved.  This
3242   // will allow deoptimization at this safepoint to find all possible
3243   // debug-info recordings, as well as let GC find all oops.
3244 
3245   oop_maps->add_gc_map( __ pc() - start, map);
3246 
3247   Label noException;
3248 
3249   __ reset_last_Java_frame(false);
3250 
3251   __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD);
3252   __ jcc(Assembler::equal, noException);
3253 
3254   // Exception pending
3255 
3256   RegisterSaver::restore_live_registers(masm, save_wide_vectors);
3257 
3258   __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3259 
3260   // No exception case
3261   __ bind(noException);
3262 
3263   Label no_adjust;
3264 #ifdef ASSERT
3265   Label bail;
3266 #endif
3267   if (!cause_return) {
3268     Label no_prefix, not_special, check_rex_prefix;
3269 
3270     // If our stashed return pc was modified by the runtime we avoid touching it
3271     __ cmpptr(rbx, Address(rbp, wordSize));
3272     __ jcc(Assembler::notEqual, no_adjust);
3273 
3274     // Skip over the poll instruction.
3275     // See NativeInstruction::is_safepoint_poll()
3276     // Possible encodings:
3277     //      85 00       test   %eax,(%rax)
3278     //      85 01       test   %eax,(%rcx)
3279     //      85 02       test   %eax,(%rdx)
3280     //      85 03       test   %eax,(%rbx)
3281     //      85 06       test   %eax,(%rsi)
3282     //      85 07       test   %eax,(%rdi)
3283     //
3284     //   41 85 00       test   %eax,(%r8)
3285     //   41 85 01       test   %eax,(%r9)
3286     //   41 85 02       test   %eax,(%r10)
3287     //   41 85 03       test   %eax,(%r11)
3288     //   41 85 06       test   %eax,(%r14)
3289     //   41 85 07       test   %eax,(%r15)
3290     //
3291     //      85 04 24    test   %eax,(%rsp)
3292     //   41 85 04 24    test   %eax,(%r12)
3293     //      85 45 00    test   %eax,0x0(%rbp)
3294     //   41 85 45 00    test   %eax,0x0(%r13)
3295     //
3296     // Notes:
3297     //  Format of legacy MAP0 test instruction:-
3298     //  [REX/REX2] [OPCODE] [ModRM] [SIB] [DISP] [IMM32]
3299     //  o  For safepoint polling instruction "test %eax,(%rax)", encoding of first register
3300     //     operand and base register of memory operand is b/w [0-8), hence we do not require
3301     //     additional REX prefix where REX.B bit stores MSB bit of register encoding, which
3302     //     is why two bytes encoding is sufficient here.
3303     //  o  For safepoint polling instruction like "test %eax,(%r8)", register encoding of BASE
3304     //     register of memory operand is 1000, thus we need additional REX prefix in this case,
3305     //     there by adding additional byte to instruction encoding.
3306     //  o  In case BASE register is one of the 32 extended GPR registers available only on targets
3307     //     supporting Intel APX extension, then we need to emit two bytes REX2 prefix to hold
3308     //     most significant two bits of 5 bit register encoding.
3309 
3310     if (VM_Version::supports_apx_f()) {
3311       __ cmpb(Address(rbx, 0), Assembler::REX2);
3312       __ jccb(Assembler::notEqual, check_rex_prefix);
3313       __ addptr(rbx, 2);
3314       __ bind(check_rex_prefix);
3315     }
3316     __ cmpb(Address(rbx, 0), NativeTstRegMem::instruction_rex_b_prefix);
3317     __ jccb(Assembler::notEqual, no_prefix);
3318     __ addptr(rbx, 1);
3319     __ bind(no_prefix);
3320 #ifdef ASSERT
3321     __ movptr(rax, rbx); // remember where 0x85 should be, for verification below
3322 #endif
3323     // r12/r13/rsp/rbp base encoding takes 3 bytes with the following register values:
3324     // r12/rsp 0x04
3325     // r13/rbp 0x05
3326     __ movzbq(rcx, Address(rbx, 1));
3327     __ andptr(rcx, 0x07); // looking for 0x04 .. 0x05
3328     __ subptr(rcx, 4);    // looking for 0x00 .. 0x01
3329     __ cmpptr(rcx, 1);
3330     __ jccb(Assembler::above, not_special);
3331     __ addptr(rbx, 1);
3332     __ bind(not_special);
3333 #ifdef ASSERT
3334     // Verify the correct encoding of the poll we're about to skip.
3335     __ cmpb(Address(rax, 0), NativeTstRegMem::instruction_code_memXregl);
3336     __ jcc(Assembler::notEqual, bail);
3337     // Mask out the modrm bits
3338     __ testb(Address(rax, 1), NativeTstRegMem::modrm_mask);
3339     // rax encodes to 0, so if the bits are nonzero it's incorrect
3340     __ jcc(Assembler::notZero, bail);
3341 #endif
3342     // Adjust return pc forward to step over the safepoint poll instruction
3343     __ addptr(rbx, 2);
3344     __ movptr(Address(rbp, wordSize), rbx);
3345   }
3346 
3347   __ bind(no_adjust);
3348   // Normal exit, restore registers and exit.
3349   RegisterSaver::restore_live_registers(masm, save_wide_vectors);
3350   __ ret(0);
3351 
3352 #ifdef ASSERT
3353   __ bind(bail);
3354   __ stop("Attempting to adjust pc to skip safepoint poll but the return point is not what we expected");
3355 #endif
3356 
3357   // Make sure all code is generated
3358   masm->flush();
3359 
3360   // Fill-out other meta info
3361   SafepointBlob* sp_blob = SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
3362 
3363   AOTCodeCache::store_code_blob(*sp_blob, AOTCodeEntry::SharedBlob, StubInfo::blob(id));
3364   return sp_blob;
3365 }
3366 
3367 //
3368 // generate_resolve_blob - call resolution (static/virtual/opt-virtual/ic-miss
3369 //
3370 // Generate a stub that calls into vm to find out the proper destination
3371 // of a java call. All the argument registers are live at this point
3372 // but since this is generic code we don't know what they are and the caller
3373 // must do any gc of the args.
3374 //
3375 RuntimeStub* SharedRuntime::generate_resolve_blob(StubId id, address destination) {
3376   assert (StubRoutines::forward_exception_entry() != nullptr, "must be generated before");
3377   assert(is_resolve_id(id), "expected a resolve stub id");
3378 
3379   const char* name = SharedRuntime::stub_name(id);
3380   CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::SharedBlob, StubInfo::blob(id));
3381   if (blob != nullptr) {
3382     return blob->as_runtime_stub();
3383   }
3384 
3385   // allocate space for the code
3386   ResourceMark rm;
3387   CodeBuffer buffer(name, 1552, 512);
3388   MacroAssembler* masm = new MacroAssembler(&buffer);
3389 
3390   int frame_size_in_words;
3391 
3392   OopMapSet *oop_maps = new OopMapSet();
3393   OopMap* map = nullptr;
3394 
3395   int start = __ offset();
3396 
3397   // No need to save vector registers since they are caller-saved anyway.
3398   map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words, /*save_wide_vectors*/ false);
3399 
3400   int frame_complete = __ offset();
3401 
3402   __ set_last_Java_frame(noreg, noreg, nullptr, rscratch1);
3403 
3404   __ mov(c_rarg0, r15_thread);
3405 
3406   __ call(RuntimeAddress(destination));
3407 
3408 
3409   // Set an oopmap for the call site.
3410   // We need this not only for callee-saved registers, but also for volatile
3411   // registers that the compiler might be keeping live across a safepoint.
3412 
3413   oop_maps->add_gc_map( __ offset() - start, map);
3414 
3415   // rax contains the address we are going to jump to assuming no exception got installed
3416 
3417   // clear last_Java_sp
3418   __ reset_last_Java_frame(false);
3419   // check for pending exceptions
3420   Label pending;
3421   __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD);
3422   __ jcc(Assembler::notEqual, pending);
3423 
3424   // get the returned Method*
3425   __ get_vm_result_metadata(rbx);
3426   __ movptr(Address(rsp, RegisterSaver::rbx_offset_in_bytes()), rbx);
3427 
3428   __ movptr(Address(rsp, RegisterSaver::rax_offset_in_bytes()), rax);
3429 
3430   RegisterSaver::restore_live_registers(masm);
3431 
3432   // We are back to the original state on entry and ready to go.
3433 
3434   __ jmp(rax);
3435 
3436   // Pending exception after the safepoint
3437 
3438   __ bind(pending);
3439 
3440   RegisterSaver::restore_live_registers(masm);
3441 
3442   // exception pending => remove activation and forward to exception handler
3443 
3444   __ movptr(Address(r15_thread, JavaThread::vm_result_oop_offset()), NULL_WORD);
3445 
3446   __ movptr(rax, Address(r15_thread, Thread::pending_exception_offset()));
3447   __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3448 
3449   // -------------
3450   // make sure all code is generated
3451   masm->flush();
3452 
3453   // return the  blob
3454   // frame_size_words or bytes??
3455   RuntimeStub* rs_blob = RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true);
3456 
3457   AOTCodeCache::store_code_blob(*rs_blob, AOTCodeEntry::SharedBlob, StubInfo::blob(id));
3458   return rs_blob;
3459 }
3460 
3461 // Continuation point for throwing of implicit exceptions that are
3462 // not handled in the current activation. Fabricates an exception
3463 // oop and initiates normal exception dispatching in this
3464 // frame. Since we need to preserve callee-saved values (currently
3465 // only for C2, but done for C1 as well) we need a callee-saved oop
3466 // map and therefore have to make these stubs into RuntimeStubs
3467 // rather than BufferBlobs.  If the compiler needs all registers to
3468 // be preserved between the fault point and the exception handler
3469 // then it must assume responsibility for that in
3470 // AbstractCompiler::continuation_for_implicit_null_exception or
3471 // continuation_for_implicit_division_by_zero_exception. All other
3472 // implicit exceptions (e.g., NullPointerException or
3473 // AbstractMethodError on entry) are either at call sites or
3474 // otherwise assume that stack unwinding will be initiated, so
3475 // caller saved registers were assumed volatile in the compiler.
3476 RuntimeStub* SharedRuntime::generate_throw_exception(StubId id, address runtime_entry) {
3477   assert(is_throw_id(id), "expected a throw stub id");
3478 
3479   const char* name = SharedRuntime::stub_name(id);
3480 
3481   // Information about frame layout at time of blocking runtime call.
3482   // Note that we only have to preserve callee-saved registers since
3483   // the compilers are responsible for supplying a continuation point
3484   // if they expect all registers to be preserved.
3485   enum layout {
3486     rbp_off = frame::arg_reg_save_area_bytes/BytesPerInt,
3487     rbp_off2,
3488     return_off,
3489     return_off2,
3490     framesize // inclusive of return address
3491   };
3492 
3493   int insts_size = 512;
3494   int locs_size  = 64;
3495 
3496   const char* timer_msg = "SharedRuntime generate_throw_exception";
3497   TraceTime timer(timer_msg, TRACETIME_LOG(Info, startuptime));
3498 
3499   CodeBlob* blob = AOTCodeCache::load_code_blob(AOTCodeEntry::SharedBlob, StubInfo::blob(id));
3500   if (blob != nullptr) {
3501     return blob->as_runtime_stub();
3502   }
3503 
3504   ResourceMark rm;
3505   CodeBuffer code(name, insts_size, locs_size);
3506   OopMapSet* oop_maps  = new OopMapSet();
3507   MacroAssembler* masm = new MacroAssembler(&code);
3508 
3509   address start = __ pc();
3510 
3511   // This is an inlined and slightly modified version of call_VM
3512   // which has the ability to fetch the return PC out of
3513   // thread-local storage and also sets up last_Java_sp slightly
3514   // differently than the real call_VM
3515 
3516   __ enter(); // required for proper stackwalking of RuntimeStub frame
3517 
3518   assert(is_even(framesize/2), "sp not 16-byte aligned");
3519 
3520   // return address and rbp are already in place
3521   __ subptr(rsp, (framesize-4) << LogBytesPerInt); // prolog
3522 
3523   int frame_complete = __ pc() - start;
3524 
3525   // Set up last_Java_sp and last_Java_fp
3526   address the_pc = __ pc();
3527   __ set_last_Java_frame(rsp, rbp, the_pc, rscratch1);
3528   __ andptr(rsp, -(StackAlignmentInBytes));    // Align stack
3529 
3530   // Call runtime
3531   __ movptr(c_rarg0, r15_thread);
3532   BLOCK_COMMENT("call runtime_entry");
3533   __ call(RuntimeAddress(runtime_entry));
3534 
3535   // Generate oop map
3536   OopMap* map = new OopMap(framesize, 0);
3537 
3538   oop_maps->add_gc_map(the_pc - start, map);
3539 
3540   __ reset_last_Java_frame(true);
3541 
3542   __ leave(); // required for proper stackwalking of RuntimeStub frame
3543 
3544   // check for pending exceptions
3545 #ifdef ASSERT
3546   Label L;
3547   __ cmpptr(Address(r15_thread, Thread::pending_exception_offset()), NULL_WORD);
3548   __ jcc(Assembler::notEqual, L);
3549   __ should_not_reach_here();
3550   __ bind(L);
3551 #endif // ASSERT
3552   __ jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3553 
3554 
3555   // codeBlob framesize is in words (not VMRegImpl::slot_size)
3556   RuntimeStub* stub =
3557     RuntimeStub::new_runtime_stub(name,
3558                                   &code,
3559                                   frame_complete,
3560                                   (framesize >> (LogBytesPerWord - LogBytesPerInt)),
3561                                   oop_maps, false);
3562   AOTCodeCache::store_code_blob(*stub, AOTCodeEntry::SharedBlob, StubInfo::blob(id));
3563 
3564   return stub;
3565 }
3566 
3567 //------------------------------Montgomery multiplication------------------------
3568 //
3569 
3570 #ifndef _WINDOWS
3571 
3572 // Subtract 0:b from carry:a.  Return carry.
3573 static julong
3574 sub(julong a[], julong b[], julong carry, long len) {
3575   long long i = 0, cnt = len;
3576   julong tmp;
3577   asm volatile("clc; "
3578                "0: ; "
3579                "mov (%[b], %[i], 8), %[tmp]; "
3580                "sbb %[tmp], (%[a], %[i], 8); "
3581                "inc %[i]; dec %[cnt]; "
3582                "jne 0b; "
3583                "mov %[carry], %[tmp]; sbb $0, %[tmp]; "
3584                : [i]"+r"(i), [cnt]"+r"(cnt), [tmp]"=&r"(tmp)
3585                : [a]"r"(a), [b]"r"(b), [carry]"r"(carry)
3586                : "memory");
3587   return tmp;
3588 }
3589 
3590 // Multiply (unsigned) Long A by Long B, accumulating the double-
3591 // length result into the accumulator formed of T0, T1, and T2.
3592 #define MACC(A, B, T0, T1, T2)                                  \
3593 do {                                                            \
3594   unsigned long hi, lo;                                         \
3595   __asm__ ("mul %5; add %%rax, %2; adc %%rdx, %3; adc $0, %4"   \
3596            : "=&d"(hi), "=a"(lo), "+r"(T0), "+r"(T1), "+g"(T2)  \
3597            : "r"(A), "a"(B) : "cc");                            \
3598  } while(0)
3599 
3600 // As above, but add twice the double-length result into the
3601 // accumulator.
3602 #define MACC2(A, B, T0, T1, T2)                                 \
3603 do {                                                            \
3604   unsigned long hi, lo;                                         \
3605   __asm__ ("mul %5; add %%rax, %2; adc %%rdx, %3; adc $0, %4; " \
3606            "add %%rax, %2; adc %%rdx, %3; adc $0, %4"           \
3607            : "=&d"(hi), "=a"(lo), "+r"(T0), "+r"(T1), "+g"(T2)  \
3608            : "r"(A), "a"(B) : "cc");                            \
3609  } while(0)
3610 
3611 #else //_WINDOWS
3612 
3613 static julong
3614 sub(julong a[], julong b[], julong carry, long len) {
3615   long i;
3616   julong tmp;
3617   unsigned char c = 1;
3618   for (i = 0; i < len; i++) {
3619     c = _addcarry_u64(c, a[i], ~b[i], &tmp);
3620     a[i] = tmp;
3621   }
3622   c = _addcarry_u64(c, carry, ~0, &tmp);
3623   return tmp;
3624 }
3625 
3626 // Multiply (unsigned) Long A by Long B, accumulating the double-
3627 // length result into the accumulator formed of T0, T1, and T2.
3628 #define MACC(A, B, T0, T1, T2)                          \
3629 do {                                                    \
3630   julong hi, lo;                            \
3631   lo = _umul128(A, B, &hi);                             \
3632   unsigned char c = _addcarry_u64(0, lo, T0, &T0);      \
3633   c = _addcarry_u64(c, hi, T1, &T1);                    \
3634   _addcarry_u64(c, T2, 0, &T2);                         \
3635  } while(0)
3636 
3637 // As above, but add twice the double-length result into the
3638 // accumulator.
3639 #define MACC2(A, B, T0, T1, T2)                         \
3640 do {                                                    \
3641   julong hi, lo;                            \
3642   lo = _umul128(A, B, &hi);                             \
3643   unsigned char c = _addcarry_u64(0, lo, T0, &T0);      \
3644   c = _addcarry_u64(c, hi, T1, &T1);                    \
3645   _addcarry_u64(c, T2, 0, &T2);                         \
3646   c = _addcarry_u64(0, lo, T0, &T0);                    \
3647   c = _addcarry_u64(c, hi, T1, &T1);                    \
3648   _addcarry_u64(c, T2, 0, &T2);                         \
3649  } while(0)
3650 
3651 #endif //_WINDOWS
3652 
3653 // Fast Montgomery multiplication.  The derivation of the algorithm is
3654 // in  A Cryptographic Library for the Motorola DSP56000,
3655 // Dusse and Kaliski, Proc. EUROCRYPT 90, pp. 230-237.
3656 
3657 static void NOINLINE
3658 montgomery_multiply(julong a[], julong b[], julong n[],
3659                     julong m[], julong inv, int len) {
3660   julong t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
3661   int i;
3662 
3663   assert(inv * n[0] == ULLONG_MAX, "broken inverse in Montgomery multiply");
3664 
3665   for (i = 0; i < len; i++) {
3666     int j;
3667     for (j = 0; j < i; j++) {
3668       MACC(a[j], b[i-j], t0, t1, t2);
3669       MACC(m[j], n[i-j], t0, t1, t2);
3670     }
3671     MACC(a[i], b[0], t0, t1, t2);
3672     m[i] = t0 * inv;
3673     MACC(m[i], n[0], t0, t1, t2);
3674 
3675     assert(t0 == 0, "broken Montgomery multiply");
3676 
3677     t0 = t1; t1 = t2; t2 = 0;
3678   }
3679 
3680   for (i = len; i < 2*len; i++) {
3681     int j;
3682     for (j = i-len+1; j < len; j++) {
3683       MACC(a[j], b[i-j], t0, t1, t2);
3684       MACC(m[j], n[i-j], t0, t1, t2);
3685     }
3686     m[i-len] = t0;
3687     t0 = t1; t1 = t2; t2 = 0;
3688   }
3689 
3690   while (t0)
3691     t0 = sub(m, n, t0, len);
3692 }
3693 
3694 // Fast Montgomery squaring.  This uses asymptotically 25% fewer
3695 // multiplies so it should be up to 25% faster than Montgomery
3696 // multiplication.  However, its loop control is more complex and it
3697 // may actually run slower on some machines.
3698 
3699 static void NOINLINE
3700 montgomery_square(julong a[], julong n[],
3701                   julong m[], julong inv, int len) {
3702   julong t0 = 0, t1 = 0, t2 = 0; // Triple-precision accumulator
3703   int i;
3704 
3705   assert(inv * n[0] == ULLONG_MAX, "broken inverse in Montgomery square");
3706 
3707   for (i = 0; i < len; i++) {
3708     int j;
3709     int end = (i+1)/2;
3710     for (j = 0; j < end; j++) {
3711       MACC2(a[j], a[i-j], t0, t1, t2);
3712       MACC(m[j], n[i-j], t0, t1, t2);
3713     }
3714     if ((i & 1) == 0) {
3715       MACC(a[j], a[j], t0, t1, t2);
3716     }
3717     for (; j < i; j++) {
3718       MACC(m[j], n[i-j], t0, t1, t2);
3719     }
3720     m[i] = t0 * inv;
3721     MACC(m[i], n[0], t0, t1, t2);
3722 
3723     assert(t0 == 0, "broken Montgomery square");
3724 
3725     t0 = t1; t1 = t2; t2 = 0;
3726   }
3727 
3728   for (i = len; i < 2*len; i++) {
3729     int start = i-len+1;
3730     int end = start + (len - start)/2;
3731     int j;
3732     for (j = start; j < end; j++) {
3733       MACC2(a[j], a[i-j], t0, t1, t2);
3734       MACC(m[j], n[i-j], t0, t1, t2);
3735     }
3736     if ((i & 1) == 0) {
3737       MACC(a[j], a[j], t0, t1, t2);
3738     }
3739     for (; j < len; j++) {
3740       MACC(m[j], n[i-j], t0, t1, t2);
3741     }
3742     m[i-len] = t0;
3743     t0 = t1; t1 = t2; t2 = 0;
3744   }
3745 
3746   while (t0)
3747     t0 = sub(m, n, t0, len);
3748 }
3749 
3750 // Swap words in a longword.
3751 static julong swap(julong x) {
3752   return (x << 32) | (x >> 32);
3753 }
3754 
3755 // Copy len longwords from s to d, word-swapping as we go.  The
3756 // destination array is reversed.
3757 static void reverse_words(julong *s, julong *d, int len) {
3758   d += len;
3759   while(len-- > 0) {
3760     d--;
3761     *d = swap(*s);
3762     s++;
3763   }
3764 }
3765 
3766 // The threshold at which squaring is advantageous was determined
3767 // experimentally on an i7-3930K (Ivy Bridge) CPU @ 3.5GHz.
3768 #define MONTGOMERY_SQUARING_THRESHOLD 64
3769 
3770 void SharedRuntime::montgomery_multiply(jint *a_ints, jint *b_ints, jint *n_ints,
3771                                         jint len, jlong inv,
3772                                         jint *m_ints) {
3773   assert(len % 2 == 0, "array length in montgomery_multiply must be even");
3774   int longwords = len/2;
3775 
3776   // Make very sure we don't use so much space that the stack might
3777   // overflow.  512 jints corresponds to an 16384-bit integer and
3778   // will use here a total of 8k bytes of stack space.
3779   int divisor = sizeof(julong) * 4;
3780   guarantee(longwords <= 8192 / divisor, "must be");
3781   int total_allocation = longwords * sizeof (julong) * 4;
3782   julong *scratch = (julong *)alloca(total_allocation);
3783 
3784   // Local scratch arrays
3785   julong
3786     *a = scratch + 0 * longwords,
3787     *b = scratch + 1 * longwords,
3788     *n = scratch + 2 * longwords,
3789     *m = scratch + 3 * longwords;
3790 
3791   reverse_words((julong *)a_ints, a, longwords);
3792   reverse_words((julong *)b_ints, b, longwords);
3793   reverse_words((julong *)n_ints, n, longwords);
3794 
3795   ::montgomery_multiply(a, b, n, m, (julong)inv, longwords);
3796 
3797   reverse_words(m, (julong *)m_ints, longwords);
3798 }
3799 
3800 void SharedRuntime::montgomery_square(jint *a_ints, jint *n_ints,
3801                                       jint len, jlong inv,
3802                                       jint *m_ints) {
3803   assert(len % 2 == 0, "array length in montgomery_square must be even");
3804   int longwords = len/2;
3805 
3806   // Make very sure we don't use so much space that the stack might
3807   // overflow.  512 jints corresponds to an 16384-bit integer and
3808   // will use here a total of 6k bytes of stack space.
3809   int divisor = sizeof(julong) * 3;
3810   guarantee(longwords <= (8192 / divisor), "must be");
3811   int total_allocation = longwords * sizeof (julong) * 3;
3812   julong *scratch = (julong *)alloca(total_allocation);
3813 
3814   // Local scratch arrays
3815   julong
3816     *a = scratch + 0 * longwords,
3817     *n = scratch + 1 * longwords,
3818     *m = scratch + 2 * longwords;
3819 
3820   reverse_words((julong *)a_ints, a, longwords);
3821   reverse_words((julong *)n_ints, n, longwords);
3822 
3823   if (len >= MONTGOMERY_SQUARING_THRESHOLD) {
3824     ::montgomery_square(a, n, m, (julong)inv, longwords);
3825   } else {
3826     ::montgomery_multiply(a, a, n, m, (julong)inv, longwords);
3827   }
3828 
3829   reverse_words(m, (julong *)m_ints, longwords);
3830 }
3831 
3832 BufferedInlineTypeBlob* SharedRuntime::generate_buffered_inline_type_adapter(const InlineKlass* vk) {
3833   BufferBlob* buf = BufferBlob::create("inline types pack/unpack", 16 * K);
3834   if (buf == nullptr) {
3835     return nullptr;
3836   }
3837   CodeBuffer buffer(buf);
3838   short buffer_locs[20];
3839   buffer.insts()->initialize_shared_locs((relocInfo*)buffer_locs,
3840                                          sizeof(buffer_locs)/sizeof(relocInfo));
3841 
3842   MacroAssembler* masm = new MacroAssembler(&buffer);
3843 
3844   const Array<SigEntry>* sig_vk = vk->extended_sig();
3845   const Array<VMRegPair>* regs = vk->return_regs();
3846 
3847   int pack_fields_jobject_off = __ offset();
3848   // Resolve pre-allocated buffer from JNI handle.
3849   // We cannot do this in generate_call_stub() because it requires GC code to be initialized.
3850   __ movptr(rax, Address(r13, 0));
3851   __ resolve_jobject(rax /* value */,
3852                      r12 /* tmp */);
3853   __ movptr(Address(r13, 0), rax);
3854 
3855   int pack_fields_off = __ offset();
3856 
3857   int j = 1;
3858   for (int i = 0; i < sig_vk->length(); i++) {
3859     BasicType bt = sig_vk->at(i)._bt;
3860     if (bt == T_METADATA) {
3861       continue;
3862     }
3863     if (bt == T_VOID) {
3864       if (sig_vk->at(i-1)._bt == T_LONG ||
3865           sig_vk->at(i-1)._bt == T_DOUBLE) {
3866         j++;
3867       }
3868       continue;
3869     }
3870     int off = sig_vk->at(i)._offset;
3871     assert(off > 0, "offset in object should be positive");
3872     VMRegPair pair = regs->at(j);
3873     VMReg r_1 = pair.first();
3874     VMReg r_2 = pair.second();
3875     Address to(rax, off);
3876     if (bt == T_FLOAT) {
3877       __ movflt(to, r_1->as_XMMRegister());
3878     } else if (bt == T_DOUBLE) {
3879       __ movdbl(to, r_1->as_XMMRegister());
3880     } else {
3881       Register val = r_1->as_Register();
3882       assert_different_registers(to.base(), val, r14, r13, rbx, rscratch1);
3883       if (is_reference_type(bt)) {
3884         __ store_heap_oop(to, val, r14, r13, rbx, IN_HEAP | ACCESS_WRITE | IS_DEST_UNINITIALIZED);
3885       } else {
3886         __ store_sized_value(to, r_1->as_Register(), type2aelembytes(bt));
3887       }
3888     }
3889     j++;
3890   }
3891   assert(j == regs->length(), "missed a field?");
3892   if (vk->has_nullable_atomic_layout()) {
3893     // Set the null marker
3894     __ movb(Address(rax, vk->null_marker_offset()), 1);
3895   }
3896   __ ret(0);
3897 
3898   int unpack_fields_off = __ offset();
3899 
3900   Label skip;
3901   Label not_null;
3902   __ testptr(rax, rax);
3903   __ jcc(Assembler::notZero, not_null);
3904 
3905   // Return value is null. Zero oop registers to make the GC happy.
3906   j = 1;
3907   for (int i = 0; i < sig_vk->length(); i++) {
3908     BasicType bt = sig_vk->at(i)._bt;
3909     if (bt == T_METADATA) {
3910       continue;
3911     }
3912     if (bt == T_VOID) {
3913       if (sig_vk->at(i-1)._bt == T_LONG ||
3914           sig_vk->at(i-1)._bt == T_DOUBLE) {
3915         j++;
3916       }
3917       continue;
3918     }
3919     if (bt == T_OBJECT || bt == T_ARRAY) {
3920       VMRegPair pair = regs->at(j);
3921       VMReg r_1 = pair.first();
3922       __ xorq(r_1->as_Register(), r_1->as_Register());
3923     }
3924     j++;
3925   }
3926   __ jmp(skip);
3927   __ bind(not_null);
3928 
3929   j = 1;
3930   for (int i = 0; i < sig_vk->length(); i++) {
3931     BasicType bt = sig_vk->at(i)._bt;
3932     if (bt == T_METADATA) {
3933       continue;
3934     }
3935     if (bt == T_VOID) {
3936       if (sig_vk->at(i-1)._bt == T_LONG ||
3937           sig_vk->at(i-1)._bt == T_DOUBLE) {
3938         j++;
3939       }
3940       continue;
3941     }
3942     int off = sig_vk->at(i)._offset;
3943     assert(off > 0, "offset in object should be positive");
3944     VMRegPair pair = regs->at(j);
3945     VMReg r_1 = pair.first();
3946     VMReg r_2 = pair.second();
3947     Address from(rax, off);
3948     if (bt == T_FLOAT) {
3949       __ movflt(r_1->as_XMMRegister(), from);
3950     } else if (bt == T_DOUBLE) {
3951       __ movdbl(r_1->as_XMMRegister(), from);
3952     } else if (bt == T_OBJECT || bt == T_ARRAY) {
3953       assert_different_registers(rax, r_1->as_Register());
3954       __ load_heap_oop(r_1->as_Register(), from);
3955     } else {
3956       assert(is_java_primitive(bt), "unexpected basic type");
3957       assert_different_registers(rax, r_1->as_Register());
3958       size_t size_in_bytes = type2aelembytes(bt);
3959       __ load_sized_value(r_1->as_Register(), from, size_in_bytes, bt != T_CHAR && bt != T_BOOLEAN);
3960     }
3961     j++;
3962   }
3963   assert(j == regs->length(), "missed a field?");
3964 
3965   __ bind(skip);
3966   __ ret(0);
3967 
3968   __ flush();
3969 
3970   return BufferedInlineTypeBlob::create(&buffer, pack_fields_off, pack_fields_jobject_off, unpack_fields_off);
3971 }
3972 
3973 #if INCLUDE_JFR
3974 
3975 // For c2: c_rarg0 is junk, call to runtime to write a checkpoint.
3976 // It returns a jobject handle to the event writer.
3977 // The handle is dereferenced and the return value is the event writer oop.
3978 RuntimeStub* SharedRuntime::generate_jfr_write_checkpoint() {
3979   enum layout {
3980     rbp_off,
3981     rbpH_off,
3982     return_off,
3983     return_off2,
3984     framesize // inclusive of return address
3985   };
3986 
3987   const char* name = SharedRuntime::stub_name(StubId::shared_jfr_write_checkpoint_id);
3988   CodeBuffer code(name, 1024, 64);
3989   MacroAssembler* masm = new MacroAssembler(&code);
3990   address start = __ pc();
3991 
3992   __ enter();
3993   address the_pc = __ pc();
3994 
3995   int frame_complete = the_pc - start;
3996 
3997   __ set_last_Java_frame(rsp, rbp, the_pc, rscratch1);
3998   __ movptr(c_rarg0, r15_thread);
3999   __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint), 1);
4000   __ reset_last_Java_frame(true);
4001 
4002   // rax is jobject handle result, unpack and process it through a barrier.
4003   __ resolve_global_jobject(rax, c_rarg0);
4004 
4005   __ leave();
4006   __ ret(0);
4007 
4008   OopMapSet* oop_maps = new OopMapSet();
4009   OopMap* map = new OopMap(framesize, 1);
4010   oop_maps->add_gc_map(frame_complete, map);
4011 
4012   RuntimeStub* stub =
4013     RuntimeStub::new_runtime_stub(name,
4014                                   &code,
4015                                   frame_complete,
4016                                   (framesize >> (LogBytesPerWord - LogBytesPerInt)),
4017                                   oop_maps,
4018                                   false);
4019   return stub;
4020 }
4021 
4022 // For c2: call to return a leased buffer.
4023 RuntimeStub* SharedRuntime::generate_jfr_return_lease() {
4024   enum layout {
4025     rbp_off,
4026     rbpH_off,
4027     return_off,
4028     return_off2,
4029     framesize // inclusive of return address
4030   };
4031 
4032   const char* name = SharedRuntime::stub_name(StubId::shared_jfr_return_lease_id);
4033   CodeBuffer code(name, 1024, 64);
4034   MacroAssembler* masm = new MacroAssembler(&code);
4035   address start = __ pc();
4036 
4037   __ enter();
4038   address the_pc = __ pc();
4039 
4040   int frame_complete = the_pc - start;
4041 
4042   __ set_last_Java_frame(rsp, rbp, the_pc, rscratch2);
4043   __ movptr(c_rarg0, r15_thread);
4044   __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), 1);
4045   __ reset_last_Java_frame(true);
4046 
4047   __ leave();
4048   __ ret(0);
4049 
4050   OopMapSet* oop_maps = new OopMapSet();
4051   OopMap* map = new OopMap(framesize, 1);
4052   oop_maps->add_gc_map(frame_complete, map);
4053 
4054   RuntimeStub* stub =
4055     RuntimeStub::new_runtime_stub(name,
4056                                   &code,
4057                                   frame_complete,
4058                                   (framesize >> (LogBytesPerWord - LogBytesPerInt)),
4059                                   oop_maps,
4060                                   false);
4061   return stub;
4062 }
4063 
4064 #endif // INCLUDE_JFR