1 /*
   2  * Copyright (c) 2008, 2025, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "asm/assembler.inline.hpp"
  26 #include "code/compiledIC.hpp"
  27 #include "code/debugInfoRec.hpp"
  28 #include "code/vtableStubs.hpp"
  29 #include "compiler/oopMap.hpp"
  30 #include "gc/shared/barrierSetAssembler.hpp"
  31 #include "interpreter/interpreter.hpp"
  32 #include "logging/log.hpp"
  33 #include "memory/resourceArea.hpp"
  34 #include "oops/klass.inline.hpp"
  35 #include "prims/methodHandles.hpp"
  36 #include "runtime/jniHandles.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/safepointMechanism.hpp"
  39 #include "runtime/stubRoutines.hpp"
  40 #include "runtime/timerTrace.hpp"
  41 #include "runtime/vframeArray.hpp"
  42 #include "utilities/align.hpp"
  43 #include "utilities/powerOfTwo.hpp"
  44 #include "vmreg_arm.inline.hpp"
  45 #ifdef COMPILER1
  46 #include "c1/c1_Runtime1.hpp"
  47 #endif
  48 #ifdef COMPILER2
  49 #include "opto/runtime.hpp"
  50 #endif
  51 
  52 #define __ masm->
  53 
  54 class RegisterSaver {
  55 public:
  56 
  57   // Special registers:
  58   //              32-bit ARM     64-bit ARM
  59   //  Rthread:       R10            R28
  60   //  LR:            R14            R30
  61 
  62   // Rthread is callee saved in the C ABI and never changed by compiled code:
  63   // no need to save it.
  64 
  65   // 2 slots for LR: the one at LR_offset and an other one at R14/R30_offset.
  66   // The one at LR_offset is a return address that is needed by stack walking.
  67   // A c2 method uses LR as a standard register so it may be live when we
  68   // branch to the runtime. The slot at R14/R30_offset is for the value of LR
  69   // in case it's live in the method we are coming from.
  70 
  71 
  72   enum RegisterLayout {
  73     fpu_save_size = FloatRegisterImpl::number_of_registers,
  74 #ifndef __SOFTFP__
  75     D0_offset = 0,
  76 #endif
  77     R0_offset = fpu_save_size,
  78     R1_offset,
  79     R2_offset,
  80     R3_offset,
  81     R4_offset,
  82     R5_offset,
  83     R6_offset,
  84 #if (FP_REG_NUM != 7)
  85     // if not saved as FP
  86     R7_offset,
  87 #endif
  88     R8_offset,
  89     R9_offset,
  90 #if (FP_REG_NUM != 11)
  91     // if not saved as FP
  92     R11_offset,
  93 #endif
  94     R12_offset,
  95     R14_offset,
  96     FP_offset,
  97     LR_offset,
  98     reg_save_size,
  99 
 100     Rmethod_offset = R9_offset,
 101     Rtemp_offset = R12_offset,
 102   };
 103 
 104   // all regs but Rthread (R10), FP (R7 or R11), SP and PC
 105   // (altFP_7_11 is the one among R7 and R11 which is not FP)
 106 #define SAVED_BASE_REGS (RegisterSet(R0, R6) | RegisterSet(R8, R9) | RegisterSet(R12) | R14 | altFP_7_11)
 107 
 108 
 109   //  When LR may be live in the nmethod from which we are coming
 110   //  then lr_saved is true, the return address is saved before the
 111   //  call to save_live_register by the caller and LR contains the
 112   //  live value.
 113 
 114   static OopMap* save_live_registers(MacroAssembler* masm,
 115                                      int* total_frame_words,
 116                                      bool lr_saved = false);
 117   static void restore_live_registers(MacroAssembler* masm, bool restore_lr = true);
 118 
 119 };
 120 
 121 
 122 
 123 
 124 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm,
 125                                            int* total_frame_words,
 126                                            bool lr_saved) {
 127   *total_frame_words = reg_save_size;
 128 
 129   OopMapSet *oop_maps = new OopMapSet();
 130   OopMap* map = new OopMap(VMRegImpl::slots_per_word * (*total_frame_words), 0);
 131 
 132   if (lr_saved) {
 133     __ push(RegisterSet(FP));
 134   } else {
 135     __ push(RegisterSet(FP) | RegisterSet(LR));
 136   }
 137   __ push(SAVED_BASE_REGS);
 138   if (HaveVFP) {
 139     if (VM_Version::has_vfp3_32()) {
 140       __ fpush(FloatRegisterSet(D16, 16));
 141     } else {
 142       if (FloatRegisterImpl::number_of_registers > 32) {
 143         assert(FloatRegisterImpl::number_of_registers == 64, "nb fp registers should be 64");
 144         __ sub(SP, SP, 32 * wordSize);
 145       }
 146     }
 147     __ fpush(FloatRegisterSet(D0, 16));
 148   } else {
 149     __ sub(SP, SP, fpu_save_size * wordSize);
 150   }
 151 
 152   int i;
 153   int j=0;
 154   for (i = R0_offset; i <= R9_offset; i++) {
 155     if (j == FP_REG_NUM) {
 156       // skip the FP register, managed below.
 157       j++;
 158     }
 159     map->set_callee_saved(VMRegImpl::stack2reg(i), as_Register(j)->as_VMReg());
 160     j++;
 161   }
 162   assert(j == R10->encoding(), "must be");
 163 #if (FP_REG_NUM != 11)
 164   // add R11, if not managed as FP
 165   map->set_callee_saved(VMRegImpl::stack2reg(R11_offset), R11->as_VMReg());
 166 #endif
 167   map->set_callee_saved(VMRegImpl::stack2reg(R12_offset), R12->as_VMReg());
 168   map->set_callee_saved(VMRegImpl::stack2reg(R14_offset), R14->as_VMReg());
 169   if (HaveVFP) {
 170     for (i = 0; i < (VM_Version::has_vfp3_32() ? 64 : 32); i+=2) {
 171       map->set_callee_saved(VMRegImpl::stack2reg(i), as_FloatRegister(i)->as_VMReg());
 172       map->set_callee_saved(VMRegImpl::stack2reg(i + 1), as_FloatRegister(i)->as_VMReg()->next());
 173     }
 174   }
 175 
 176   return map;
 177 }
 178 
 179 void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_lr) {
 180   if (HaveVFP) {
 181     __ fpop(FloatRegisterSet(D0, 16));
 182     if (VM_Version::has_vfp3_32()) {
 183       __ fpop(FloatRegisterSet(D16, 16));
 184     } else {
 185       if (FloatRegisterImpl::number_of_registers > 32) {
 186         assert(FloatRegisterImpl::number_of_registers == 64, "nb fp registers should be 64");
 187         __ add(SP, SP, 32 * wordSize);
 188       }
 189     }
 190   } else {
 191     __ add(SP, SP, fpu_save_size * wordSize);
 192   }
 193   __ pop(SAVED_BASE_REGS);
 194   if (restore_lr) {
 195     __ pop(RegisterSet(FP) | RegisterSet(LR));
 196   } else {
 197     __ pop(RegisterSet(FP));
 198   }
 199 }
 200 
 201 
 202 static void push_result_registers(MacroAssembler* masm, BasicType ret_type) {
 203 #ifdef __ABI_HARD__
 204   if (ret_type == T_DOUBLE || ret_type == T_FLOAT) {
 205     __ sub(SP, SP, 8);
 206     __ fstd(D0, Address(SP));
 207     return;
 208   }
 209 #endif // __ABI_HARD__
 210   __ raw_push(R0, R1);
 211 }
 212 
 213 static void pop_result_registers(MacroAssembler* masm, BasicType ret_type) {
 214 #ifdef __ABI_HARD__
 215   if (ret_type == T_DOUBLE || ret_type == T_FLOAT) {
 216     __ fldd(D0, Address(SP));
 217     __ add(SP, SP, 8);
 218     return;
 219   }
 220 #endif // __ABI_HARD__
 221   __ raw_pop(R0, R1);
 222 }
 223 
 224 static void push_param_registers(MacroAssembler* masm, int fp_regs_in_arguments) {
 225   // R1-R3 arguments need to be saved, but we push 4 registers for 8-byte alignment
 226   __ push(RegisterSet(R0, R3));
 227 
 228   // preserve arguments
 229   // Likely not needed as the locking code won't probably modify volatile FP registers,
 230   // but there is no way to guarantee that
 231   if (fp_regs_in_arguments) {
 232     // convert fp_regs_in_arguments to a number of double registers
 233     int double_regs_num = (fp_regs_in_arguments + 1) >> 1;
 234     __ fpush_hardfp(FloatRegisterSet(D0, double_regs_num));
 235   }
 236 }
 237 
 238 static void pop_param_registers(MacroAssembler* masm, int fp_regs_in_arguments) {
 239   if (fp_regs_in_arguments) {
 240     int double_regs_num = (fp_regs_in_arguments + 1) >> 1;
 241     __ fpop_hardfp(FloatRegisterSet(D0, double_regs_num));
 242   }
 243   __ pop(RegisterSet(R0, R3));
 244 }
 245 
 246 
 247 
 248 // Is vector's size (in bytes) bigger than a size saved by default?
 249 // All vector registers are saved by default on ARM.
 250 bool SharedRuntime::is_wide_vector(int size) {
 251   return false;
 252 }
 253 
 254 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
 255                                         VMRegPair *regs,
 256                                         int total_args_passed) {
 257   int slot = 0;
 258   int ireg = 0;
 259 #ifdef __ABI_HARD__
 260   int fp_slot = 0;
 261   int single_fpr_slot = 0;
 262 #endif // __ABI_HARD__
 263   for (int i = 0; i < total_args_passed; i++) {
 264     switch (sig_bt[i]) {
 265     case T_SHORT:
 266     case T_CHAR:
 267     case T_BYTE:
 268     case T_BOOLEAN:
 269     case T_INT:
 270     case T_ARRAY:
 271     case T_OBJECT:
 272     case T_ADDRESS:
 273     case T_METADATA:
 274 #ifndef __ABI_HARD__
 275     case T_FLOAT:
 276 #endif // !__ABI_HARD__
 277       if (ireg < 4) {
 278         Register r = as_Register(ireg);
 279         regs[i].set1(r->as_VMReg());
 280         ireg++;
 281       } else {
 282         regs[i].set1(VMRegImpl::stack2reg(slot));
 283         slot++;
 284       }
 285       break;
 286     case T_LONG:
 287 #ifndef __ABI_HARD__
 288     case T_DOUBLE:
 289 #endif // !__ABI_HARD__
 290       assert((i + 1) < total_args_passed && sig_bt[i+1] == T_VOID, "missing Half" );
 291       if (ireg <= 2) {
 292 #if (ALIGN_WIDE_ARGUMENTS == 1)
 293         if(ireg & 1) ireg++;  // Aligned location required
 294 #endif
 295         Register r1 = as_Register(ireg);
 296         Register r2 = as_Register(ireg + 1);
 297         regs[i].set_pair(r2->as_VMReg(), r1->as_VMReg());
 298         ireg += 2;
 299 #if (ALIGN_WIDE_ARGUMENTS == 0)
 300       } else if (ireg == 3) {
 301         // uses R3 + one stack slot
 302         Register r = as_Register(ireg);
 303         regs[i].set_pair(VMRegImpl::stack2reg(slot), r->as_VMReg());
 304         ireg += 1;
 305         slot += 1;
 306 #endif
 307       } else {
 308         if (slot & 1) slot++; // Aligned location required
 309         regs[i].set_pair(VMRegImpl::stack2reg(slot+1), VMRegImpl::stack2reg(slot));
 310         slot += 2;
 311         ireg = 4;
 312       }
 313       break;
 314     case T_VOID:
 315       regs[i].set_bad();
 316       break;
 317 #ifdef __ABI_HARD__
 318     case T_FLOAT:
 319       if ((fp_slot < 16)||(single_fpr_slot & 1)) {
 320         if ((single_fpr_slot & 1) == 0) {
 321           single_fpr_slot = fp_slot;
 322           fp_slot += 2;
 323         }
 324         FloatRegister r = as_FloatRegister(single_fpr_slot);
 325         single_fpr_slot++;
 326         regs[i].set1(r->as_VMReg());
 327       } else {
 328         regs[i].set1(VMRegImpl::stack2reg(slot));
 329         slot++;
 330       }
 331       break;
 332     case T_DOUBLE:
 333       assert(ALIGN_WIDE_ARGUMENTS == 1, "ABI_HARD not supported with unaligned wide arguments");
 334       if (fp_slot <= 14) {
 335         FloatRegister r1 = as_FloatRegister(fp_slot);
 336         FloatRegister r2 = as_FloatRegister(fp_slot+1);
 337         regs[i].set_pair(r2->as_VMReg(), r1->as_VMReg());
 338         fp_slot += 2;
 339       } else {
 340         if(slot & 1) slot++;
 341         regs[i].set_pair(VMRegImpl::stack2reg(slot+1), VMRegImpl::stack2reg(slot));
 342         slot += 2;
 343         single_fpr_slot = 16;
 344       }
 345       break;
 346 #endif // __ABI_HARD__
 347     default:
 348       ShouldNotReachHere();
 349     }
 350   }
 351   return slot;
 352 }
 353 
 354 int SharedRuntime::vector_calling_convention(VMRegPair *regs,
 355                                              uint num_bits,
 356                                              uint total_args_passed) {
 357   Unimplemented();
 358   return 0;
 359 }
 360 
 361 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
 362                                            VMRegPair *regs,
 363                                            int total_args_passed) {
 364 #ifdef __SOFTFP__
 365   // soft float is the same as the C calling convention.
 366   return c_calling_convention(sig_bt, regs, nullptr, total_args_passed);
 367 #endif // __SOFTFP__
 368   int slot = 0;
 369   int ireg = 0;
 370   int freg = 0;
 371   int single_fpr = 0;
 372 
 373   for (int i = 0; i < total_args_passed; i++) {
 374     switch (sig_bt[i]) {
 375     case T_SHORT:
 376     case T_CHAR:
 377     case T_BYTE:
 378     case T_BOOLEAN:
 379     case T_INT:
 380     case T_ARRAY:
 381     case T_OBJECT:
 382     case T_ADDRESS:
 383       if (ireg < 4) {
 384         Register r = as_Register(ireg++);
 385         regs[i].set1(r->as_VMReg());
 386       } else {
 387         regs[i].set1(VMRegImpl::stack2reg(slot++));
 388       }
 389       break;
 390     case T_FLOAT:
 391       // C2 utilizes S14/S15 for mem-mem moves
 392       if ((freg < 16 COMPILER2_PRESENT(-2)) || (single_fpr & 1)) {
 393         if ((single_fpr & 1) == 0) {
 394           single_fpr = freg;
 395           freg += 2;
 396         }
 397         FloatRegister r = as_FloatRegister(single_fpr++);
 398         regs[i].set1(r->as_VMReg());
 399       } else {
 400         regs[i].set1(VMRegImpl::stack2reg(slot++));
 401       }
 402       break;
 403     case T_DOUBLE:
 404       // C2 utilizes S14/S15 for mem-mem moves
 405       if (freg <= 14 COMPILER2_PRESENT(-2)) {
 406         FloatRegister r1 = as_FloatRegister(freg);
 407         FloatRegister r2 = as_FloatRegister(freg + 1);
 408         regs[i].set_pair(r2->as_VMReg(), r1->as_VMReg());
 409         freg += 2;
 410       } else {
 411         // Keep internally the aligned calling convention,
 412         // ignoring ALIGN_WIDE_ARGUMENTS
 413         if (slot & 1) slot++;
 414         regs[i].set_pair(VMRegImpl::stack2reg(slot + 1), VMRegImpl::stack2reg(slot));
 415         slot += 2;
 416         single_fpr = 16;
 417       }
 418       break;
 419     case T_LONG:
 420       // Keep internally the aligned calling convention,
 421       // ignoring ALIGN_WIDE_ARGUMENTS
 422       if (ireg <= 2) {
 423         if (ireg & 1) ireg++;
 424         Register r1 = as_Register(ireg);
 425         Register r2 = as_Register(ireg + 1);
 426         regs[i].set_pair(r2->as_VMReg(), r1->as_VMReg());
 427         ireg += 2;
 428       } else {
 429         if (slot & 1) slot++;
 430         regs[i].set_pair(VMRegImpl::stack2reg(slot + 1), VMRegImpl::stack2reg(slot));
 431         slot += 2;
 432         ireg = 4;
 433       }
 434       break;
 435     case T_VOID:
 436       regs[i].set_bad();
 437       break;
 438     default:
 439       ShouldNotReachHere();
 440     }
 441   }
 442 
 443   return slot;
 444 }
 445 
 446 static void patch_callers_callsite(MacroAssembler *masm) {
 447   Label skip;
 448 
 449   __ ldr(Rtemp, Address(Rmethod, Method::code_offset()));
 450   __ cbz(Rtemp, skip);
 451 
 452   // Pushing an even number of registers for stack alignment.
 453   // Selecting R9, which had to be saved anyway for some platforms.
 454   __ push(RegisterSet(R0, R3) | R9 | LR);
 455   __ fpush_hardfp(FloatRegisterSet(D0, 8));
 456 
 457   __ mov(R0, Rmethod);
 458   __ mov(R1, LR);
 459   __ call(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite));
 460 
 461   __ fpop_hardfp(FloatRegisterSet(D0, 8));
 462   __ pop(RegisterSet(R0, R3) | R9 | LR);
 463 
 464   __ bind(skip);
 465 }
 466 
 467 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
 468                                     int total_args_passed, int comp_args_on_stack,
 469                                     const BasicType *sig_bt, const VMRegPair *regs) {
 470   // TODO: ARM - May be can use ldm to load arguments
 471   const Register tmp = Rtemp; // avoid erasing R5_mh
 472 
 473   // Next assert may not be needed but safer. Extra analysis required
 474   // if this there is not enough free registers and we need to use R5 here.
 475   assert_different_registers(tmp, R5_mh);
 476 
 477   // 6243940 We might end up in handle_wrong_method if
 478   // the callee is deoptimized as we race thru here. If that
 479   // happens we don't want to take a safepoint because the
 480   // caller frame will look interpreted and arguments are now
 481   // "compiled" so it is much better to make this transition
 482   // invisible to the stack walking code. Unfortunately if
 483   // we try and find the callee by normal means a safepoint
 484   // is possible. So we stash the desired callee in the thread
 485   // and the vm will find there should this case occur.
 486   Address callee_target_addr(Rthread, JavaThread::callee_target_offset());
 487   __ str(Rmethod, callee_target_addr);
 488 
 489 
 490   assert_different_registers(tmp, R0, R1, R2, R3, Rsender_sp, Rmethod);
 491 
 492   const Register initial_sp = Rmethod; // temporarily scratched
 493 
 494   // Old code was modifying R4 but this looks unsafe (particularly with JSR292)
 495   assert_different_registers(tmp, R0, R1, R2, R3, Rsender_sp, initial_sp);
 496 
 497   __ mov(initial_sp, SP);
 498 
 499   if (comp_args_on_stack) {
 500     __ sub_slow(SP, SP, comp_args_on_stack * VMRegImpl::stack_slot_size);
 501   }
 502   __ bic(SP, SP, StackAlignmentInBytes - 1);
 503 
 504   for (int i = 0; i < total_args_passed; i++) {
 505     if (sig_bt[i] == T_VOID) {
 506       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 507       continue;
 508     }
 509     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(), "must be ordered");
 510     int arg_offset = Interpreter::expr_offset_in_bytes(total_args_passed - 1 - i);
 511 
 512     VMReg r_1 = regs[i].first();
 513     VMReg r_2 = regs[i].second();
 514     if (r_1->is_stack()) {
 515       int stack_offset = r_1->reg2stack() * VMRegImpl::stack_slot_size;
 516       if (!r_2->is_valid()) {
 517         __ ldr(tmp, Address(initial_sp, arg_offset));
 518         __ str(tmp, Address(SP, stack_offset));
 519       } else {
 520         __ ldr(tmp, Address(initial_sp, arg_offset - Interpreter::stackElementSize));
 521         __ str(tmp, Address(SP, stack_offset));
 522         __ ldr(tmp, Address(initial_sp, arg_offset));
 523         __ str(tmp, Address(SP, stack_offset + wordSize));
 524       }
 525     } else if (r_1->is_Register()) {
 526       if (!r_2->is_valid()) {
 527         __ ldr(r_1->as_Register(), Address(initial_sp, arg_offset));
 528       } else {
 529         __ ldr(r_1->as_Register(), Address(initial_sp, arg_offset - Interpreter::stackElementSize));
 530         __ ldr(r_2->as_Register(), Address(initial_sp, arg_offset));
 531       }
 532     } else if (r_1->is_FloatRegister()) {
 533 #ifdef __SOFTFP__
 534       ShouldNotReachHere();
 535 #endif // __SOFTFP__
 536       if (!r_2->is_valid()) {
 537         __ flds(r_1->as_FloatRegister(), Address(initial_sp, arg_offset));
 538       } else {
 539         __ fldd(r_1->as_FloatRegister(), Address(initial_sp, arg_offset - Interpreter::stackElementSize));
 540       }
 541     } else {
 542       assert(!r_1->is_valid() && !r_2->is_valid(), "must be");
 543     }
 544   }
 545 
 546   // restore Rmethod (scratched for initial_sp)
 547   __ ldr(Rmethod, callee_target_addr);
 548   __ ldr(PC, Address(Rmethod, Method::from_compiled_offset()));
 549 
 550 }
 551 
 552 static void gen_c2i_adapter(MacroAssembler *masm,
 553                             int total_args_passed,  int comp_args_on_stack,
 554                             const BasicType *sig_bt, const VMRegPair *regs,
 555                             Label& skip_fixup) {
 556   // TODO: ARM - May be can use stm to deoptimize arguments
 557   const Register tmp = Rtemp;
 558 
 559   patch_callers_callsite(masm);
 560   __ bind(skip_fixup);
 561 
 562   __ mov(Rsender_sp, SP); // not yet saved
 563 
 564 
 565   int extraspace = total_args_passed * Interpreter::stackElementSize;
 566   if (extraspace) {
 567     __ sub_slow(SP, SP, extraspace);
 568   }
 569 
 570   for (int i = 0; i < total_args_passed; i++) {
 571     if (sig_bt[i] == T_VOID) {
 572       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 573       continue;
 574     }
 575     int stack_offset = (total_args_passed - 1 - i) * Interpreter::stackElementSize;
 576 
 577     VMReg r_1 = regs[i].first();
 578     VMReg r_2 = regs[i].second();
 579     if (r_1->is_stack()) {
 580       int arg_offset = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 581       if (!r_2->is_valid()) {
 582         __ ldr(tmp, Address(SP, arg_offset));
 583         __ str(tmp, Address(SP, stack_offset));
 584       } else {
 585         __ ldr(tmp, Address(SP, arg_offset));
 586         __ str(tmp, Address(SP, stack_offset - Interpreter::stackElementSize));
 587         __ ldr(tmp, Address(SP, arg_offset + wordSize));
 588         __ str(tmp, Address(SP, stack_offset));
 589       }
 590     } else if (r_1->is_Register()) {
 591       if (!r_2->is_valid()) {
 592         __ str(r_1->as_Register(), Address(SP, stack_offset));
 593       } else {
 594         __ str(r_1->as_Register(), Address(SP, stack_offset - Interpreter::stackElementSize));
 595         __ str(r_2->as_Register(), Address(SP, stack_offset));
 596       }
 597     } else if (r_1->is_FloatRegister()) {
 598 #ifdef __SOFTFP__
 599       ShouldNotReachHere();
 600 #endif // __SOFTFP__
 601       if (!r_2->is_valid()) {
 602         __ fsts(r_1->as_FloatRegister(), Address(SP, stack_offset));
 603       } else {
 604         __ fstd(r_1->as_FloatRegister(), Address(SP, stack_offset - Interpreter::stackElementSize));
 605       }
 606     } else {
 607       assert(!r_1->is_valid() && !r_2->is_valid(), "must be");
 608     }
 609   }
 610 
 611   __ ldr(PC, Address(Rmethod, Method::interpreter_entry_offset()));
 612 
 613 }
 614 
 615 void SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
 616                                             int total_args_passed,
 617                                             int comp_args_on_stack,
 618                                             const BasicType *sig_bt,
 619                                             const VMRegPair *regs,
 620                                             AdapterHandlerEntry* handler) {
 621   address i2c_entry = __ pc();
 622   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 623 
 624   address c2i_unverified_entry = __ pc();
 625   Label skip_fixup;
 626   const Register receiver       = R0;
 627   const Register holder_klass   = Rtemp; // XXX should be OK for C2 but not 100% sure
 628 
 629   __ ic_check(1 /* end_alignment */);
 630   __ ldr(Rmethod, Address(Ricklass, CompiledICData::speculated_method_offset()));
 631 
 632   __ ldr(Rtemp, Address(Rmethod, Method::code_offset()), eq);
 633   __ cmp(Rtemp, 0, eq);
 634   __ b(skip_fixup, eq);
 635   __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, noreg, ne);
 636 
 637   address c2i_entry = __ pc();
 638   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
 639 
 640   handler->set_entry_points(i2c_entry, c2i_entry, c2i_unverified_entry, nullptr);
 641   return;
 642 }
 643 
 644 
 645 static int reg2offset_in(VMReg r) {
 646   // Account for saved FP and LR
 647   return r->reg2stack() * VMRegImpl::stack_slot_size + 2*wordSize;
 648 }
 649 
 650 static int reg2offset_out(VMReg r) {
 651   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 652 }
 653 
 654 
 655 static void verify_oop_args(MacroAssembler* masm,
 656                             const methodHandle& method,
 657                             const BasicType* sig_bt,
 658                             const VMRegPair* regs) {
 659   Register temp_reg = Rmethod;  // not part of any compiled calling seq
 660   if (VerifyOops) {
 661     for (int i = 0; i < method->size_of_parameters(); i++) {
 662       if (sig_bt[i] == T_OBJECT || sig_bt[i] == T_ARRAY) {
 663         VMReg r = regs[i].first();
 664         assert(r->is_valid(), "bad oop arg");
 665         if (r->is_stack()) {
 666           __ ldr(temp_reg, Address(SP, r->reg2stack() * VMRegImpl::stack_slot_size));
 667           __ verify_oop(temp_reg);
 668         } else {
 669           __ verify_oop(r->as_Register());
 670         }
 671       }
 672     }
 673   }
 674 }
 675 
 676 static void gen_special_dispatch(MacroAssembler* masm,
 677                                  const methodHandle& method,
 678                                  const BasicType* sig_bt,
 679                                  const VMRegPair* regs) {
 680   verify_oop_args(masm, method, sig_bt, regs);
 681   vmIntrinsics::ID iid = method->intrinsic_id();
 682 
 683   // Now write the args into the outgoing interpreter space
 684   bool     has_receiver   = false;
 685   Register receiver_reg   = noreg;
 686   int      member_arg_pos = -1;
 687   Register member_reg     = noreg;
 688   int      ref_kind       = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
 689   if (ref_kind != 0) {
 690     member_arg_pos = method->size_of_parameters() - 1;  // trailing MemberName argument
 691     member_reg = Rmethod;  // known to be free at this point
 692     has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
 693   } else if (iid == vmIntrinsics::_invokeBasic) {
 694     has_receiver = true;
 695   } else {
 696     fatal("unexpected intrinsic id %d", vmIntrinsics::as_int(iid));
 697   }
 698 
 699   if (member_reg != noreg) {
 700     // Load the member_arg into register, if necessary.
 701     SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
 702     VMReg r = regs[member_arg_pos].first();
 703     if (r->is_stack()) {
 704       __ ldr(member_reg, Address(SP, r->reg2stack() * VMRegImpl::stack_slot_size));
 705     } else {
 706       // no data motion is needed
 707       member_reg = r->as_Register();
 708     }
 709   }
 710 
 711   if (has_receiver) {
 712     // Make sure the receiver is loaded into a register.
 713     assert(method->size_of_parameters() > 0, "oob");
 714     assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
 715     VMReg r = regs[0].first();
 716     assert(r->is_valid(), "bad receiver arg");
 717     if (r->is_stack()) {
 718       // Porting note:  This assumes that compiled calling conventions always
 719       // pass the receiver oop in a register.  If this is not true on some
 720       // platform, pick a temp and load the receiver from stack.
 721       assert(false, "receiver always in a register");
 722       receiver_reg = j_rarg0;  // known to be free at this point
 723       __ ldr(receiver_reg, Address(SP, r->reg2stack() * VMRegImpl::stack_slot_size));
 724     } else {
 725       // no data motion is needed
 726       receiver_reg = r->as_Register();
 727     }
 728   }
 729 
 730   // Figure out which address we are really jumping to:
 731   MethodHandles::generate_method_handle_dispatch(masm, iid,
 732                                                  receiver_reg, member_reg, /*for_compiler_entry:*/ true);
 733 }
 734 
 735 // ---------------------------------------------------------------------------
 736 // Generate a native wrapper for a given method.  The method takes arguments
 737 // in the Java compiled code convention, marshals them to the native
 738 // convention (handlizes oops, etc), transitions to native, makes the call,
 739 // returns to java state (possibly blocking), unhandlizes any result and
 740 // returns.
 741 nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 742                                                 const methodHandle& method,
 743                                                 int compile_id,
 744                                                 BasicType* in_sig_bt,
 745                                                 VMRegPair* in_regs,
 746                                                 BasicType ret_type) {
 747   if (method->is_method_handle_intrinsic()) {
 748     vmIntrinsics::ID iid = method->intrinsic_id();
 749     intptr_t start = (intptr_t)__ pc();
 750     int vep_offset = ((intptr_t)__ pc()) - start;
 751     gen_special_dispatch(masm,
 752                          method,
 753                          in_sig_bt,
 754                          in_regs);
 755     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
 756     __ flush();
 757     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
 758     return nmethod::new_native_nmethod(method,
 759                                        compile_id,
 760                                        masm->code(),
 761                                        vep_offset,
 762                                        frame_complete,
 763                                        stack_slots / VMRegImpl::slots_per_word,
 764                                        in_ByteSize(-1),
 765                                        in_ByteSize(-1),
 766                                        (OopMapSet*)nullptr);
 767   }
 768   // Arguments for JNI method include JNIEnv and Class if static
 769 
 770   // Usage of Rtemp should be OK since scratched by native call
 771 
 772   bool method_is_static = method->is_static();
 773 
 774   const int total_in_args = method->size_of_parameters();
 775   int total_c_args = total_in_args + (method_is_static ? 2 : 1);
 776 
 777   BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
 778   VMRegPair* out_regs   = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
 779 
 780   int argc = 0;
 781   out_sig_bt[argc++] = T_ADDRESS;
 782   if (method_is_static) {
 783     out_sig_bt[argc++] = T_OBJECT;
 784   }
 785 
 786   int i;
 787   for (i = 0; i < total_in_args; i++) {
 788     out_sig_bt[argc++] = in_sig_bt[i];
 789   }
 790 
 791   int out_arg_slots = c_calling_convention(out_sig_bt, out_regs, total_c_args);
 792   int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots;
 793   // Since object arguments need to be wrapped, we must preserve space
 794   // for those object arguments which come in registers (GPR_PARAMS maximum)
 795   // plus one more slot for Klass handle (for static methods)
 796   int oop_handle_offset = stack_slots;
 797   stack_slots += (GPR_PARAMS + 1) * VMRegImpl::slots_per_word;
 798 
 799   // Plus a lock if needed
 800   int lock_slot_offset = 0;
 801   if (method->is_synchronized()) {
 802     lock_slot_offset = stack_slots;
 803     assert(sizeof(BasicLock) == wordSize, "adjust this code");
 804     stack_slots += VMRegImpl::slots_per_word;
 805   }
 806 
 807   // Space to save return address and FP
 808   stack_slots += 2 * VMRegImpl::slots_per_word;
 809 
 810   // Calculate the final stack size taking account of alignment
 811   stack_slots = align_up(stack_slots, StackAlignmentInBytes / VMRegImpl::stack_slot_size);
 812   int stack_size = stack_slots * VMRegImpl::stack_slot_size;
 813   int lock_slot_fp_offset = stack_size - 2 * wordSize -
 814     lock_slot_offset * VMRegImpl::stack_slot_size;
 815 
 816   // Unverified entry point
 817   address start = __ pc();
 818 
 819   const Register receiver = R0; // see receiverOpr()
 820   __ verify_oop(receiver);
 821   // Inline cache check
 822   __ ic_check(CodeEntryAlignment /* end_alignment */);
 823 
 824   // Verified entry point
 825   int vep_offset = __ pc() - start;
 826 
 827   if ((InlineObjectHash && method->intrinsic_id() == vmIntrinsics::_hashCode) || (method->intrinsic_id() == vmIntrinsics::_identityHashCode)) {
 828     // Object.hashCode, System.identityHashCode can pull the hashCode from the header word
 829     // instead of doing a full VM transition once it's been computed.
 830     Label slow_case;
 831     const Register obj_reg = R0;
 832 
 833     // Unlike for Object.hashCode, System.identityHashCode is static method and
 834     // gets object as argument instead of the receiver.
 835     if (method->intrinsic_id() == vmIntrinsics::_identityHashCode) {
 836       assert(method->is_static(), "method should be static");
 837       // return 0 for null reference input, return val = R0 = obj_reg = 0
 838       __ cmp(obj_reg, 0);
 839       __ bx(LR, eq);
 840     }
 841 
 842     __ ldr(Rtemp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
 843 
 844     assert(markWord::unlocked_value == 1, "adjust this code");
 845     __ tbz(Rtemp, exact_log2(markWord::unlocked_value), slow_case);
 846 
 847     __ bics(Rtemp, Rtemp, ~markWord::hash_mask_in_place);
 848     __ mov(R0, AsmOperand(Rtemp, lsr, markWord::hash_shift), ne);
 849     __ bx(LR, ne);
 850 
 851     __ bind(slow_case);
 852   }
 853 
 854   // Bang stack pages
 855   __ arm_stack_overflow_check(stack_size, Rtemp);
 856 
 857   // Setup frame linkage
 858   __ raw_push(FP, LR);
 859   __ mov(FP, SP);
 860   __ sub_slow(SP, SP, stack_size - 2*wordSize);
 861 
 862   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 863   assert(bs != nullptr, "Sanity");
 864   bs->nmethod_entry_barrier(masm);
 865 
 866   int frame_complete = __ pc() - start;
 867 
 868   OopMapSet* oop_maps = new OopMapSet();
 869   OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
 870   const int extra_args = method_is_static ? 2 : 1;
 871   int receiver_offset = -1;
 872   int fp_regs_in_arguments = 0;
 873 
 874   for (i = total_in_args; --i >= 0; ) {
 875     switch (in_sig_bt[i]) {
 876     case T_ARRAY:
 877     case T_OBJECT: {
 878       VMReg src = in_regs[i].first();
 879       VMReg dst = out_regs[i + extra_args].first();
 880       if (src->is_stack()) {
 881         assert(dst->is_stack(), "must be");
 882         assert(i != 0, "Incoming receiver is always in a register");
 883         __ ldr(Rtemp, Address(FP, reg2offset_in(src)));
 884         __ cmp(Rtemp, 0);
 885         __ add(Rtemp, FP, reg2offset_in(src), ne);
 886         __ str(Rtemp, Address(SP, reg2offset_out(dst)));
 887         int offset_in_older_frame = src->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 888         map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + stack_slots));
 889       } else {
 890         int offset = oop_handle_offset * VMRegImpl::stack_slot_size;
 891         __ str(src->as_Register(), Address(SP, offset));
 892         map->set_oop(VMRegImpl::stack2reg(oop_handle_offset));
 893         if ((i == 0) && (!method_is_static)) {
 894           receiver_offset = offset;
 895         }
 896         oop_handle_offset += VMRegImpl::slots_per_word;
 897 
 898         if (dst->is_stack()) {
 899           __ movs(Rtemp, src->as_Register());
 900           __ add(Rtemp, SP, offset, ne);
 901           __ str(Rtemp, Address(SP, reg2offset_out(dst)));
 902         } else {
 903           __ movs(dst->as_Register(), src->as_Register());
 904           __ add(dst->as_Register(), SP, offset, ne);
 905         }
 906       }
 907     }
 908 
 909     case T_VOID:
 910       break;
 911 
 912 
 913 #ifdef __SOFTFP__
 914     case T_DOUBLE:
 915 #endif
 916     case T_LONG: {
 917       VMReg src_1 = in_regs[i].first();
 918       VMReg src_2 = in_regs[i].second();
 919       VMReg dst_1 = out_regs[i + extra_args].first();
 920       VMReg dst_2 = out_regs[i + extra_args].second();
 921 #if (ALIGN_WIDE_ARGUMENTS == 0)
 922       // C convention can mix a register and a stack slot for a
 923       // 64-bits native argument.
 924 
 925       // Note: following code should work independently of whether
 926       // the Java calling convention follows C convention or whether
 927       // it aligns 64-bit values.
 928       if (dst_2->is_Register()) {
 929         if (src_1->as_Register() != dst_1->as_Register()) {
 930           assert(src_1->as_Register() != dst_2->as_Register() &&
 931                  src_2->as_Register() != dst_2->as_Register(), "must be");
 932           __ mov(dst_2->as_Register(), src_2->as_Register());
 933           __ mov(dst_1->as_Register(), src_1->as_Register());
 934         } else {
 935           assert(src_2->as_Register() == dst_2->as_Register(), "must be");
 936         }
 937       } else if (src_2->is_Register()) {
 938         if (dst_1->is_Register()) {
 939           // dst mixes a register and a stack slot
 940           assert(dst_2->is_stack() && src_1->is_Register() && src_2->is_Register(), "must be");
 941           assert(src_1->as_Register() != dst_1->as_Register(), "must be");
 942           __ str(src_2->as_Register(), Address(SP, reg2offset_out(dst_2)));
 943           __ mov(dst_1->as_Register(), src_1->as_Register());
 944         } else {
 945           // registers to stack slots
 946           assert(dst_2->is_stack() && src_1->is_Register() && src_2->is_Register(), "must be");
 947           __ str(src_1->as_Register(), Address(SP, reg2offset_out(dst_1)));
 948           __ str(src_2->as_Register(), Address(SP, reg2offset_out(dst_2)));
 949         }
 950       } else if (src_1->is_Register()) {
 951         if (dst_1->is_Register()) {
 952           // src and dst must be R3 + stack slot
 953           assert(dst_1->as_Register() == src_1->as_Register(), "must be");
 954           __ ldr(Rtemp,    Address(FP, reg2offset_in(src_2)));
 955           __ str(Rtemp,    Address(SP, reg2offset_out(dst_2)));
 956         } else {
 957           // <R3,stack> -> <stack,stack>
 958           assert(dst_2->is_stack() && src_2->is_stack(), "must be");
 959           __ ldr(LR, Address(FP, reg2offset_in(src_2)));
 960           __ str(src_1->as_Register(), Address(SP, reg2offset_out(dst_1)));
 961           __ str(LR, Address(SP, reg2offset_out(dst_2)));
 962         }
 963       } else {
 964         assert(src_2->is_stack() && dst_1->is_stack() && dst_2->is_stack(), "must be");
 965         __ ldr(Rtemp, Address(FP, reg2offset_in(src_1)));
 966         __ ldr(LR,    Address(FP, reg2offset_in(src_2)));
 967         __ str(Rtemp, Address(SP, reg2offset_out(dst_1)));
 968         __ str(LR,    Address(SP, reg2offset_out(dst_2)));
 969       }
 970 #else // ALIGN_WIDE_ARGUMENTS
 971       if (src_1->is_stack()) {
 972         assert(src_2->is_stack() && dst_1->is_stack() && dst_2->is_stack(), "must be");
 973         __ ldr(Rtemp, Address(FP, reg2offset_in(src_1)));
 974         __ ldr(LR,    Address(FP, reg2offset_in(src_2)));
 975         __ str(Rtemp, Address(SP, reg2offset_out(dst_1)));
 976         __ str(LR,    Address(SP, reg2offset_out(dst_2)));
 977       } else if (dst_1->is_stack()) {
 978         assert(dst_2->is_stack() && src_1->is_Register() && src_2->is_Register(), "must be");
 979         __ str(src_1->as_Register(), Address(SP, reg2offset_out(dst_1)));
 980         __ str(src_2->as_Register(), Address(SP, reg2offset_out(dst_2)));
 981       } else if (src_1->as_Register() == dst_1->as_Register()) {
 982         assert(src_2->as_Register() == dst_2->as_Register(), "must be");
 983       } else {
 984         assert(src_1->as_Register() != dst_2->as_Register() &&
 985                src_2->as_Register() != dst_2->as_Register(), "must be");
 986         __ mov(dst_2->as_Register(), src_2->as_Register());
 987         __ mov(dst_1->as_Register(), src_1->as_Register());
 988       }
 989 #endif // ALIGN_WIDE_ARGUMENTS
 990       break;
 991     }
 992 
 993 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
 994     case T_FLOAT: {
 995       VMReg src = in_regs[i].first();
 996       VMReg dst = out_regs[i + extra_args].first();
 997       if (src->is_stack()) {
 998         assert(dst->is_stack(), "must be");
 999         __ ldr(Rtemp, Address(FP, reg2offset_in(src)));
1000         __ str(Rtemp, Address(SP, reg2offset_out(dst)));
1001       } else if (dst->is_stack()) {
1002         __ fsts(src->as_FloatRegister(), Address(SP, reg2offset_out(dst)));
1003       } else {
1004         assert(src->is_FloatRegister() && dst->is_Register(), "must be");
1005         __ fmrs(dst->as_Register(), src->as_FloatRegister());
1006       }
1007       break;
1008     }
1009 
1010     case T_DOUBLE: {
1011       VMReg src_1 = in_regs[i].first();
1012       VMReg src_2 = in_regs[i].second();
1013       VMReg dst_1 = out_regs[i + extra_args].first();
1014       VMReg dst_2 = out_regs[i + extra_args].second();
1015       if (src_1->is_stack()) {
1016         assert(src_2->is_stack() && dst_1->is_stack() && dst_2->is_stack(), "must be");
1017         __ ldr(Rtemp, Address(FP, reg2offset_in(src_1)));
1018         __ ldr(LR,    Address(FP, reg2offset_in(src_2)));
1019         __ str(Rtemp, Address(SP, reg2offset_out(dst_1)));
1020         __ str(LR,    Address(SP, reg2offset_out(dst_2)));
1021       } else if (dst_1->is_stack()) {
1022         assert(dst_2->is_stack() && src_1->is_FloatRegister(), "must be");
1023         __ fstd(src_1->as_FloatRegister(), Address(SP, reg2offset_out(dst_1)));
1024 #if (ALIGN_WIDE_ARGUMENTS == 0)
1025       } else if (dst_2->is_stack()) {
1026         assert(! src_2->is_stack(), "must be"); // assuming internal java convention is aligned
1027         // double register must go into R3 + one stack slot
1028         __ fmrrd(dst_1->as_Register(), Rtemp, src_1->as_FloatRegister());
1029         __ str(Rtemp, Address(SP, reg2offset_out(dst_2)));
1030 #endif
1031       } else {
1032         assert(src_1->is_FloatRegister() && dst_1->is_Register() && dst_2->is_Register(), "must be");
1033         __ fmrrd(dst_1->as_Register(), dst_2->as_Register(), src_1->as_FloatRegister());
1034       }
1035       break;
1036     }
1037 #endif // __SOFTFP__
1038 
1039 #ifdef __ABI_HARD__
1040     case T_FLOAT: {
1041       VMReg src = in_regs[i].first();
1042       VMReg dst = out_regs[i + extra_args].first();
1043       if (src->is_stack()) {
1044         if (dst->is_stack()) {
1045           __ ldr(Rtemp, Address(FP, reg2offset_in(src)));
1046           __ str(Rtemp, Address(SP, reg2offset_out(dst)));
1047         } else {
1048           // C2 Java calling convention does not populate S14 and S15, therefore
1049           // those need to be loaded from stack here
1050           __ flds(dst->as_FloatRegister(), Address(FP, reg2offset_in(src)));
1051           fp_regs_in_arguments++;
1052         }
1053       } else {
1054         assert(src->is_FloatRegister(), "must be");
1055         fp_regs_in_arguments++;
1056       }
1057       break;
1058     }
1059     case T_DOUBLE: {
1060       VMReg src_1 = in_regs[i].first();
1061       VMReg src_2 = in_regs[i].second();
1062       VMReg dst_1 = out_regs[i + extra_args].first();
1063       VMReg dst_2 = out_regs[i + extra_args].second();
1064       if (src_1->is_stack()) {
1065         if (dst_1->is_stack()) {
1066           assert(dst_2->is_stack(), "must be");
1067           __ ldr(Rtemp, Address(FP, reg2offset_in(src_1)));
1068           __ ldr(LR,    Address(FP, reg2offset_in(src_2)));
1069           __ str(Rtemp, Address(SP, reg2offset_out(dst_1)));
1070           __ str(LR,    Address(SP, reg2offset_out(dst_2)));
1071         } else {
1072           // C2 Java calling convention does not populate S14 and S15, therefore
1073           // those need to be loaded from stack here
1074           __ fldd(dst_1->as_FloatRegister(), Address(FP, reg2offset_in(src_1)));
1075           fp_regs_in_arguments += 2;
1076         }
1077       } else {
1078         assert(src_1->is_FloatRegister() && src_2->is_FloatRegister(), "must be");
1079         fp_regs_in_arguments += 2;
1080       }
1081       break;
1082     }
1083 #endif // __ABI_HARD__
1084 
1085     default: {
1086       assert(in_sig_bt[i] != T_ADDRESS, "found T_ADDRESS in java args");
1087       VMReg src = in_regs[i].first();
1088       VMReg dst = out_regs[i + extra_args].first();
1089       if (src->is_stack()) {
1090         assert(dst->is_stack(), "must be");
1091         __ ldr(Rtemp, Address(FP, reg2offset_in(src)));
1092         __ str(Rtemp, Address(SP, reg2offset_out(dst)));
1093       } else if (dst->is_stack()) {
1094         __ str(src->as_Register(), Address(SP, reg2offset_out(dst)));
1095       } else {
1096         assert(src->is_Register() && dst->is_Register(), "must be");
1097         __ mov(dst->as_Register(), src->as_Register());
1098       }
1099     }
1100     }
1101   }
1102 
1103   // Get Klass mirror
1104   int klass_offset = -1;
1105   if (method_is_static) {
1106     klass_offset = oop_handle_offset * VMRegImpl::stack_slot_size;
1107     __ mov_oop(Rtemp, JNIHandles::make_local(method->method_holder()->java_mirror()));
1108     __ add(c_rarg1, SP, klass_offset);
1109     __ str(Rtemp, Address(SP, klass_offset));
1110     map->set_oop(VMRegImpl::stack2reg(oop_handle_offset));
1111   }
1112 
1113   // the PC offset given to add_gc_map must match the PC saved in set_last_Java_frame
1114   int pc_offset = __ set_last_Java_frame(SP, FP, true, Rtemp);
1115   assert(((__ pc()) - start) == __ offset(), "warning: start differs from code_begin");
1116   oop_maps->add_gc_map(pc_offset, map);
1117 
1118   // Order last_Java_pc store with the thread state transition (to _thread_in_native)
1119   __ membar(MacroAssembler::StoreStore, Rtemp);
1120 
1121   // RedefineClasses() tracing support for obsolete method entry
1122   if (log_is_enabled(Trace, redefine, class, obsolete)) {
1123     __ save_caller_save_registers();
1124     __ mov(R0, Rthread);
1125     __ mov_metadata(R1, method());
1126     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry), R0, R1);
1127     __ restore_caller_save_registers();
1128   }
1129 
1130   const Register sync_handle = R5;
1131   const Register sync_obj    = R6;
1132   const Register disp_hdr    = altFP_7_11;
1133   const Register tmp         = R8;
1134 
1135   Label slow_lock, lock_done, fast_lock;
1136   if (method->is_synchronized()) {
1137     // The first argument is a handle to sync object (a class or an instance)
1138     __ ldr(sync_obj, Address(R1));
1139     // Remember the handle for the unlocking code
1140     __ mov(sync_handle, R1);
1141 
1142     if (LockingMode == LM_LIGHTWEIGHT) {
1143       log_trace(fastlock)("SharedRuntime lock fast");
1144       __ lightweight_lock(sync_obj /* object */, disp_hdr /* t1 */, tmp /* t2 */, Rtemp /* t3 */,
1145                           0x7 /* savemask */, slow_lock);
1146       // Fall through to lock_done
1147     } else if (LockingMode == LM_LEGACY) {
1148       const Register mark = tmp;
1149       // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
1150       // That would be acceptable as either CAS or slow case path is taken in that case
1151 
1152       __ ldr(mark, Address(sync_obj, oopDesc::mark_offset_in_bytes()));
1153       __ sub(disp_hdr, FP, lock_slot_fp_offset);
1154       __ tst(mark, markWord::unlocked_value);
1155       __ b(fast_lock, ne);
1156 
1157       // Check for recursive lock
1158       // See comments in InterpreterMacroAssembler::lock_object for
1159       // explanations on the fast recursive locking check.
1160       // Check independently the low bits and the distance to SP
1161       // -1- test low 2 bits
1162       __ movs(Rtemp, AsmOperand(mark, lsl, 30));
1163       // -2- test (hdr - SP) if the low two bits are 0
1164       __ sub(Rtemp, mark, SP, eq);
1165       __ movs(Rtemp, AsmOperand(Rtemp, lsr, exact_log2(os::vm_page_size())), eq);
1166       // If still 'eq' then recursive locking OK
1167       // set to zero if recursive lock, set to non zero otherwise (see discussion in JDK-8267042)
1168       __ str(Rtemp, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()));
1169       __ b(lock_done, eq);
1170       __ b(slow_lock);
1171 
1172       __ bind(fast_lock);
1173       __ str(mark, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()));
1174 
1175       __ cas_for_lock_acquire(mark, disp_hdr, sync_obj, Rtemp, slow_lock);
1176     }
1177     __ bind(lock_done);
1178   }
1179 
1180   // Get JNIEnv*
1181   __ add(c_rarg0, Rthread, in_bytes(JavaThread::jni_environment_offset()));
1182 
1183   // Perform thread state transition
1184   __ mov(Rtemp, _thread_in_native);
1185   __ str(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1186 
1187   // Finally, call the native method
1188   __ call(method->native_function());
1189 
1190   // Set FPSCR/FPCR to a known state
1191   if (AlwaysRestoreFPU) {
1192     __ restore_default_fp_mode();
1193   }
1194 
1195   // Ensure a Boolean result is mapped to 0..1
1196   if (ret_type == T_BOOLEAN) {
1197     __ c2bool(R0);
1198   }
1199 
1200   // Do a safepoint check while thread is in transition state
1201   Label call_safepoint_runtime, return_to_java;
1202   __ mov(Rtemp, _thread_in_native_trans);
1203   __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1204 
1205   // make sure the store is observed before reading the SafepointSynchronize state and further mem refs
1206   if (!UseSystemMemoryBarrier) {
1207     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp);
1208   }
1209 
1210   __ safepoint_poll(R2, call_safepoint_runtime);
1211   __ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
1212   __ cmp(R3, 0);
1213   __ b(call_safepoint_runtime, ne);
1214 
1215   __ bind(return_to_java);
1216 
1217   // Perform thread state transition and reguard stack yellow pages if needed
1218   Label reguard, reguard_done;
1219   __ mov(Rtemp, _thread_in_Java);
1220   __ ldr_s32(R2, Address(Rthread, JavaThread::stack_guard_state_offset()));
1221   __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1222 
1223   __ cmp(R2, StackOverflow::stack_guard_yellow_reserved_disabled);
1224   __ b(reguard, eq);
1225   __ bind(reguard_done);
1226 
1227   Label slow_unlock, unlock_done;
1228   if (method->is_synchronized()) {
1229     if (LockingMode == LM_LIGHTWEIGHT) {
1230       log_trace(fastlock)("SharedRuntime unlock fast");
1231       __ lightweight_unlock(sync_obj, R2 /* t1 */, tmp /* t2 */, Rtemp /* t3 */,
1232                             7 /* savemask */, slow_unlock);
1233       // Fall through
1234     } else if (LockingMode == LM_LEGACY) {
1235       // See C1_MacroAssembler::unlock_object() for more comments
1236       __ ldr(sync_obj, Address(sync_handle));
1237 
1238       // See C1_MacroAssembler::unlock_object() for more comments
1239       __ ldr(R2, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()));
1240       __ cbz(R2, unlock_done);
1241 
1242       __ cas_for_lock_release(disp_hdr, R2, sync_obj, Rtemp, slow_unlock);
1243     }
1244     __ bind(unlock_done);
1245   }
1246 
1247   // Set last java frame and handle block to zero
1248   __ ldr(LR, Address(Rthread, JavaThread::active_handles_offset()));
1249   __ reset_last_Java_frame(Rtemp); // sets Rtemp to 0 on 32-bit ARM
1250 
1251   __ str_32(Rtemp, Address(LR, JNIHandleBlock::top_offset()));
1252   if (CheckJNICalls) {
1253     __ str(__ zero_register(Rtemp), Address(Rthread, JavaThread::pending_jni_exception_check_fn_offset()));
1254   }
1255 
1256   // Unbox oop result, e.g. JNIHandles::resolve value in R0.
1257   if (ret_type == T_OBJECT || ret_type == T_ARRAY) {
1258     __ resolve_jobject(R0,      // value
1259                        Rtemp,   // tmp1
1260                        R1_tmp); // tmp2
1261   }
1262 
1263   // Any exception pending?
1264   __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
1265   __ mov(SP, FP);
1266 
1267   __ cmp(Rtemp, 0);
1268   // Pop the frame and return if no exception pending
1269   __ pop(RegisterSet(FP) | RegisterSet(PC), eq);
1270   // Pop the frame and forward the exception. Rexception_pc contains return address.
1271   __ ldr(FP, Address(SP, wordSize, post_indexed), ne);
1272   __ ldr(Rexception_pc, Address(SP, wordSize, post_indexed), ne);
1273   __ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
1274 
1275   // Safepoint operation and/or pending suspend request is in progress.
1276   // Save the return values and call the runtime function by hand.
1277   __ bind(call_safepoint_runtime);
1278   push_result_registers(masm, ret_type);
1279   __ mov(R0, Rthread);
1280   __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
1281   pop_result_registers(masm, ret_type);
1282   __ b(return_to_java);
1283 
1284   // Reguard stack pages. Save native results around a call to C runtime.
1285   __ bind(reguard);
1286   push_result_registers(masm, ret_type);
1287   __ call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
1288   pop_result_registers(masm, ret_type);
1289   __ b(reguard_done);
1290 
1291   if (method->is_synchronized()) {
1292     // Locking slow case
1293     __ bind(slow_lock);
1294 
1295     push_param_registers(masm, fp_regs_in_arguments);
1296 
1297     // last_Java_frame is already set, so do call_VM manually; no exception can occur
1298     __ mov(R0, sync_obj);
1299     __ mov(R1, disp_hdr);
1300     __ mov(R2, Rthread);
1301     __ call(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C));
1302 
1303     pop_param_registers(masm, fp_regs_in_arguments);
1304 
1305     __ b(lock_done);
1306 
1307     // Unlocking slow case
1308     __ bind(slow_unlock);
1309 
1310     push_result_registers(masm, ret_type);
1311 
1312     // Clear pending exception before reentering VM.
1313     // Can store the oop in register since it is a leaf call.
1314     assert_different_registers(Rtmp_save1, sync_obj, disp_hdr);
1315     __ ldr(Rtmp_save1, Address(Rthread, Thread::pending_exception_offset()));
1316     Register zero = __ zero_register(Rtemp);
1317     __ str(zero, Address(Rthread, Thread::pending_exception_offset()));
1318     __ mov(R0, sync_obj);
1319     __ mov(R1, disp_hdr);
1320     __ mov(R2, Rthread);
1321     __ call(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C));
1322     __ str(Rtmp_save1, Address(Rthread, Thread::pending_exception_offset()));
1323 
1324     pop_result_registers(masm, ret_type);
1325 
1326     __ b(unlock_done);
1327   }
1328 
1329   __ flush();
1330   return nmethod::new_native_nmethod(method,
1331                                      compile_id,
1332                                      masm->code(),
1333                                      vep_offset,
1334                                      frame_complete,
1335                                      stack_slots / VMRegImpl::slots_per_word,
1336                                      in_ByteSize(method_is_static ? klass_offset : receiver_offset),
1337                                      in_ByteSize(lock_slot_offset * VMRegImpl::stack_slot_size),
1338                                      oop_maps);
1339 }
1340 
1341 // this function returns the adjust size (in number of words) to a c2i adapter
1342 // activation for use during deoptimization
1343 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals) {
1344   int extra_locals_size = (callee_locals - callee_parameters) * Interpreter::stackElementWords;
1345   return extra_locals_size;
1346 }
1347 
1348 
1349 // Number of stack slots between incoming argument block and the start of
1350 // a new frame.  The PROLOG must add this many slots to the stack.  The
1351 // EPILOG must remove this many slots.
1352 // FP + LR
1353 uint SharedRuntime::in_preserve_stack_slots() {
1354   return 2 * VMRegImpl::slots_per_word;
1355 }
1356 
1357 uint SharedRuntime::out_preserve_stack_slots() {
1358   return 0;
1359 }
1360 
1361 VMReg SharedRuntime::thread_register() {
1362   Unimplemented();
1363   return nullptr;
1364 }
1365 
1366 //------------------------------generate_deopt_blob----------------------------
1367 void SharedRuntime::generate_deopt_blob() {
1368   ResourceMark rm;
1369   const char* name = SharedRuntime::stub_name(SharedStubId::deopt_id);
1370   CodeBuffer buffer(name, 1024, 1024);
1371   int frame_size_in_words;
1372   OopMapSet* oop_maps;
1373   int reexecute_offset;
1374   int exception_in_tls_offset;
1375   int exception_offset;
1376 
1377   MacroAssembler* masm = new MacroAssembler(&buffer);
1378   Label cont;
1379   const Register Rkind   = R9; // caller-saved
1380   const Register Rublock = R6;
1381   const Register Rsender = altFP_7_11;
1382   assert_different_registers(Rkind, Rublock, Rsender, Rexception_obj, Rexception_pc, R0, R1, R2, R3, R8, Rtemp);
1383 
1384   address start = __ pc();
1385 
1386   oop_maps = new OopMapSet();
1387   // LR saved by caller (can be live in c2 method)
1388 
1389   // A deopt is a case where LR may be live in the c2 nmethod. So it's
1390   // not possible to call the deopt blob from the nmethod and pass the
1391   // address of the deopt handler of the nmethod in LR. What happens
1392   // now is that the caller of the deopt blob pushes the current
1393   // address so the deopt blob doesn't have to do it. This way LR can
1394   // be preserved, contains the live value from the nmethod and is
1395   // saved at R14/R30_offset here.
1396   OopMap* map = RegisterSaver::save_live_registers(masm, &frame_size_in_words, true);
1397   __ mov(Rkind, Deoptimization::Unpack_deopt);
1398   __ b(cont);
1399 
1400   exception_offset = __ pc() - start;
1401 
1402   // Transfer Rexception_obj & Rexception_pc in TLS and fall thru to the
1403   // exception_in_tls_offset entry point.
1404   __ str(Rexception_obj, Address(Rthread, JavaThread::exception_oop_offset()));
1405   __ str(Rexception_pc, Address(Rthread, JavaThread::exception_pc_offset()));
1406   // Force return value to null to avoid confusing the escape analysis
1407   // logic. Everything is dead here anyway.
1408   __ mov(R0, 0);
1409 
1410   exception_in_tls_offset = __ pc() - start;
1411 
1412   // Exception data is in JavaThread structure
1413   // Patch the return address of the current frame
1414   __ ldr(LR, Address(Rthread, JavaThread::exception_pc_offset()));
1415   (void) RegisterSaver::save_live_registers(masm, &frame_size_in_words);
1416   {
1417     const Register Rzero = __ zero_register(Rtemp); // XXX should be OK for C2 but not 100% sure
1418     __ str(Rzero, Address(Rthread, JavaThread::exception_pc_offset()));
1419   }
1420   __ mov(Rkind, Deoptimization::Unpack_exception);
1421   __ b(cont);
1422 
1423   reexecute_offset = __ pc() - start;
1424 
1425   (void) RegisterSaver::save_live_registers(masm, &frame_size_in_words);
1426   __ mov(Rkind, Deoptimization::Unpack_reexecute);
1427 
1428   // Calculate UnrollBlock and save the result in Rublock
1429   __ bind(cont);
1430   __ mov(R0, Rthread);
1431   __ mov(R1, Rkind);
1432 
1433   int pc_offset = __ set_last_Java_frame(SP, FP, false, Rtemp); // note: FP may not need to be saved (not on x86)
1434   assert(((__ pc()) - start) == __ offset(), "warning: start differs from code_begin");
1435   __ call(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info));
1436   if (pc_offset == -1) {
1437     pc_offset = __ offset();
1438   }
1439   oop_maps->add_gc_map(pc_offset, map);
1440   __ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call
1441 
1442   __ mov(Rublock, R0);
1443 
1444   // Reload Rkind from the UnrollBlock (might have changed)
1445   __ ldr_s32(Rkind, Address(Rublock, Deoptimization::UnrollBlock::unpack_kind_offset()));
1446   Label noException;
1447   __ cmp_32(Rkind, Deoptimization::Unpack_exception);   // Was exception pending?
1448   __ b(noException, ne);
1449   // handle exception case
1450 #ifdef ASSERT
1451   // assert that exception_pc is zero in tls
1452   { Label L;
1453     __ ldr(Rexception_pc, Address(Rthread, JavaThread::exception_pc_offset()));
1454     __ cbz(Rexception_pc, L);
1455     __ stop("exception pc should be null");
1456     __ bind(L);
1457   }
1458 #endif
1459   __ ldr(Rexception_obj, Address(Rthread, JavaThread::exception_oop_offset()));
1460   __ verify_oop(Rexception_obj);
1461   {
1462     const Register Rzero = __ zero_register(Rtemp);
1463     __ str(Rzero, Address(Rthread, JavaThread::exception_oop_offset()));
1464   }
1465 
1466   __ bind(noException);
1467 
1468   // This frame is going away.  Fetch return value, so we can move it to
1469   // a new frame.
1470   __ ldr(R0, Address(SP, RegisterSaver::R0_offset * wordSize));
1471   __ ldr(R1, Address(SP, RegisterSaver::R1_offset * wordSize));
1472 #ifndef __SOFTFP__
1473   __ ldr_double(D0, Address(SP, RegisterSaver::D0_offset * wordSize));
1474 #endif
1475   // pop frame
1476   __ add(SP, SP, RegisterSaver::reg_save_size * wordSize);
1477 
1478   // Set initial stack state before pushing interpreter frames
1479   __ ldr_s32(Rtemp, Address(Rublock, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset()));
1480   __ ldr(R2, Address(Rublock, Deoptimization::UnrollBlock::frame_pcs_offset()));
1481   __ ldr(R3, Address(Rublock, Deoptimization::UnrollBlock::frame_sizes_offset()));
1482 
1483   __ add(SP, SP, Rtemp);
1484 
1485 #ifdef ASSERT
1486   // Compilers generate code that bang the stack by as much as the
1487   // interpreter would need. So this stack banging should never
1488   // trigger a fault. Verify that it does not on non product builds.
1489   // See if it is enough stack to push deoptimized frames.
1490   //
1491   // The compiled method that we are deoptimizing was popped from the stack.
1492   // If the stack bang results in a stack overflow, we don't return to the
1493   // method that is being deoptimized. The stack overflow exception is
1494   // propagated to the caller of the deoptimized method. Need to get the pc
1495   // from the caller in LR and restore FP.
1496   __ ldr(LR, Address(R2, 0));
1497   __ ldr(FP, Address(Rublock, Deoptimization::UnrollBlock::initial_info_offset()));
1498   __ ldr_s32(R8, Address(Rublock, Deoptimization::UnrollBlock::total_frame_sizes_offset()));
1499   __ arm_stack_overflow_check(R8, Rtemp);
1500 #endif
1501   __ ldr_s32(R8, Address(Rublock, Deoptimization::UnrollBlock::number_of_frames_offset()));
1502 
1503   // Pick up the initial fp we should save
1504   // XXX Note: was ldr(FP, Address(FP));
1505 
1506   // The compiler no longer uses FP as a frame pointer for the
1507   // compiled code. It can be used by the allocator in C2 or to
1508   // memorize the original SP for JSR292 call sites.
1509 
1510   // Hence, ldr(FP, Address(FP)) is probably not correct. For x86,
1511   // Deoptimization::fetch_unroll_info computes the right FP value and
1512   // stores it in Rublock.initial_info. This has been activated for ARM.
1513   __ ldr(FP, Address(Rublock, Deoptimization::UnrollBlock::initial_info_offset()));
1514 
1515   __ ldr_s32(Rtemp, Address(Rublock, Deoptimization::UnrollBlock::caller_adjustment_offset()));
1516   __ mov(Rsender, SP);
1517   __ sub(SP, SP, Rtemp);
1518 
1519   // Push interpreter frames in a loop
1520   Label loop;
1521   __ bind(loop);
1522   __ ldr(LR, Address(R2, wordSize, post_indexed));         // load frame pc
1523   __ ldr(Rtemp, Address(R3, wordSize, post_indexed));      // load frame size
1524 
1525   __ raw_push(FP, LR);                                     // create new frame
1526   __ mov(FP, SP);
1527   __ sub(Rtemp, Rtemp, 2*wordSize);
1528 
1529   __ sub(SP, SP, Rtemp);
1530 
1531   __ str(Rsender, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize));
1532   __ mov(LR, 0);
1533   __ str(LR, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1534 
1535   __ subs(R8, R8, 1);                               // decrement counter
1536   __ mov(Rsender, SP);
1537   __ b(loop, ne);
1538 
1539   // Re-push self-frame
1540   __ ldr(LR, Address(R2));
1541   __ raw_push(FP, LR);
1542   __ mov(FP, SP);
1543   __ sub(SP, SP, (frame_size_in_words - 2) * wordSize);
1544 
1545   // Restore frame locals after moving the frame
1546   __ str(R0, Address(SP, RegisterSaver::R0_offset * wordSize));
1547   __ str(R1, Address(SP, RegisterSaver::R1_offset * wordSize));
1548 
1549 #ifndef __SOFTFP__
1550   __ str_double(D0, Address(SP, RegisterSaver::D0_offset * wordSize));
1551 #endif // !__SOFTFP__
1552 
1553 #ifdef ASSERT
1554   // Reload Rkind from the UnrollBlock and check that it was not overwritten (Rkind is not callee-saved)
1555   { Label L;
1556     __ ldr_s32(Rtemp, Address(Rublock, Deoptimization::UnrollBlock::unpack_kind_offset()));
1557     __ cmp_32(Rkind, Rtemp);
1558     __ b(L, eq);
1559     __ stop("Rkind was overwritten");
1560     __ bind(L);
1561   }
1562 #endif
1563 
1564   // Call unpack_frames with proper arguments
1565   __ mov(R0, Rthread);
1566   __ mov(R1, Rkind);
1567 
1568   pc_offset = __ set_last_Java_frame(SP, FP, true, Rtemp);
1569   assert(((__ pc()) - start) == __ offset(), "warning: start differs from code_begin");
1570   __ call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames));
1571   if (pc_offset == -1) {
1572     pc_offset = __ offset();
1573   }
1574   oop_maps->add_gc_map(pc_offset, new OopMap(frame_size_in_words * VMRegImpl::slots_per_word, 0));
1575   __ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call
1576 
1577   // Collect return values, pop self-frame and jump to interpreter
1578   __ ldr(R0, Address(SP, RegisterSaver::R0_offset * wordSize));
1579   __ ldr(R1, Address(SP, RegisterSaver::R1_offset * wordSize));
1580   // Interpreter floats controlled by __SOFTFP__, but compiler
1581   // float return value registers controlled by __ABI_HARD__
1582   // This matters for vfp-sflt builds.
1583 #ifndef __SOFTFP__
1584   // Interpreter hard float
1585 #ifdef __ABI_HARD__
1586   // Compiler float return value in FP registers
1587   __ ldr_double(D0, Address(SP, RegisterSaver::D0_offset * wordSize));
1588 #else
1589   // Compiler float return value in integer registers,
1590   // copy to D0 for interpreter (S0 <-- R0)
1591   __ fmdrr(D0_tos, R0, R1);
1592 #endif
1593 #endif // !__SOFTFP__
1594   __ mov(SP, FP);
1595 
1596   __ pop(RegisterSet(FP) | RegisterSet(PC));
1597 
1598   __ flush();
1599 
1600   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset,
1601                                            reexecute_offset, frame_size_in_words);
1602   _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
1603 }
1604 
1605 //------------------------------generate_handler_blob------
1606 //
1607 // Generate a special Compile2Runtime blob that saves all registers,
1608 // setup oopmap, and calls safepoint code to stop the compiled code for
1609 // a safepoint.
1610 //
1611 SafepointBlob* SharedRuntime::generate_handler_blob(SharedStubId id, address call_ptr) {
1612   assert(StubRoutines::forward_exception_entry() != nullptr, "must be generated before");
1613   assert(is_polling_page_id(id), "expected a polling page stub id");
1614 
1615   ResourceMark rm;
1616   const char* name = SharedRuntime::stub_name(id);
1617   CodeBuffer buffer(name, 256, 256);
1618   int frame_size_words;
1619   OopMapSet* oop_maps;
1620 
1621   bool cause_return = (id == SharedStubId::polling_page_return_handler_id);
1622 
1623   MacroAssembler* masm = new MacroAssembler(&buffer);
1624   address start = __ pc();
1625   oop_maps = new OopMapSet();
1626 
1627   if (!cause_return) {
1628     __ sub(SP, SP, 4); // make room for LR which may still be live
1629                        // here if we are coming from a c2 method
1630   }
1631 
1632   OopMap* map = RegisterSaver::save_live_registers(masm, &frame_size_words, !cause_return);
1633   if (!cause_return) {
1634     // update saved PC with correct value
1635     // need 2 steps because LR can be live in c2 method
1636     __ ldr(LR, Address(Rthread, JavaThread::saved_exception_pc_offset()));
1637     __ str(LR, Address(SP, RegisterSaver::LR_offset * wordSize));
1638   }
1639 
1640   __ mov(R0, Rthread);
1641   int pc_offset = __ set_last_Java_frame(SP, FP, false, Rtemp); // note: FP may not need to be saved (not on x86)
1642   assert(((__ pc()) - start) == __ offset(), "warning: start differs from code_begin");
1643   __ call(call_ptr);
1644   if (pc_offset == -1) {
1645     pc_offset = __ offset();
1646   }
1647   oop_maps->add_gc_map(pc_offset, map);
1648   __ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call
1649 
1650   if (!cause_return) {
1651     // If our stashed return pc was modified by the runtime we avoid touching it
1652     __ ldr(R3_tmp, Address(Rthread, JavaThread::saved_exception_pc_offset()));
1653     __ ldr(R2_tmp, Address(SP, RegisterSaver::LR_offset * wordSize));
1654     __ cmp(R2_tmp, R3_tmp);
1655     // Adjust return pc forward to step over the safepoint poll instruction
1656     __ add(R2_tmp, R2_tmp, 4, eq);
1657     __ str(R2_tmp, Address(SP, RegisterSaver::LR_offset * wordSize), eq);
1658 
1659     // Check for pending exception
1660     __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
1661     __ cmp(Rtemp, 0);
1662 
1663     RegisterSaver::restore_live_registers(masm, false);
1664     __ pop(PC, eq);
1665     __ pop(Rexception_pc);
1666   } else {
1667     // Check for pending exception
1668     __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
1669     __ cmp(Rtemp, 0);
1670 
1671     RegisterSaver::restore_live_registers(masm);
1672     __ bx(LR, eq);
1673     __ mov(Rexception_pc, LR);
1674   }
1675 
1676   __ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
1677 
1678   __ flush();
1679 
1680   return SafepointBlob::create(&buffer, oop_maps, frame_size_words);
1681 }
1682 
1683 RuntimeStub* SharedRuntime::generate_resolve_blob(SharedStubId id, address destination) {
1684   assert(StubRoutines::forward_exception_entry() != nullptr, "must be generated before");
1685   assert(is_resolve_id(id), "expected a resolve stub id");
1686 
1687   ResourceMark rm;
1688   const char* name = SharedRuntime::stub_name(id);
1689   CodeBuffer buffer(name, 1000, 512);
1690   int frame_size_words;
1691   OopMapSet *oop_maps;
1692   int frame_complete;
1693 
1694   MacroAssembler* masm = new MacroAssembler(&buffer);
1695   Label pending_exception;
1696 
1697   int start = __ offset();
1698 
1699   oop_maps = new OopMapSet();
1700   OopMap* map = RegisterSaver::save_live_registers(masm, &frame_size_words);
1701 
1702   frame_complete = __ offset();
1703 
1704   __ mov(R0, Rthread);
1705 
1706   int pc_offset = __ set_last_Java_frame(SP, FP, false, Rtemp);
1707   assert(start == 0, "warning: start differs from code_begin");
1708   __ call(destination);
1709   if (pc_offset == -1) {
1710     pc_offset = __ offset();
1711   }
1712   oop_maps->add_gc_map(pc_offset, map);
1713   __ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call
1714 
1715   __ ldr(R1, Address(Rthread, Thread::pending_exception_offset()));
1716   __ cbnz(R1, pending_exception);
1717 
1718   // Overwrite saved register values
1719 
1720   // Place metadata result of VM call into Rmethod
1721   __ get_vm_result_2(R1, Rtemp);
1722   __ str(R1, Address(SP, RegisterSaver::Rmethod_offset * wordSize));
1723 
1724   // Place target address (VM call result) into Rtemp
1725   __ str(R0, Address(SP, RegisterSaver::Rtemp_offset * wordSize));
1726 
1727   RegisterSaver::restore_live_registers(masm);
1728   __ jump(Rtemp);
1729 
1730   __ bind(pending_exception);
1731 
1732   RegisterSaver::restore_live_registers(masm);
1733   const Register Rzero = __ zero_register(Rtemp);
1734   __ str(Rzero, Address(Rthread, JavaThread::vm_result_2_offset()));
1735   __ mov(Rexception_pc, LR);
1736   __ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
1737 
1738   __ flush();
1739 
1740   return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_words, oop_maps, true);
1741 }
1742 
1743 //------------------------------------------------------------------------------------------------------------------------
1744 // Continuation point for throwing of implicit exceptions that are not handled in
1745 // the current activation. Fabricates an exception oop and initiates normal
1746 // exception dispatching in this frame.
1747 RuntimeStub* SharedRuntime::generate_throw_exception(SharedStubId id, address runtime_entry) {
1748   assert(is_throw_id(id), "expected a throw stub id");
1749 
1750   const char* name = SharedRuntime::stub_name(id);
1751 
1752   int insts_size = 128;
1753   int locs_size  = 32;
1754 
1755   ResourceMark rm;
1756   const char* timer_msg = "SharedRuntime generate_throw_exception";
1757   TraceTime timer(timer_msg, TRACETIME_LOG(Info, startuptime));
1758 
1759   CodeBuffer code(name, insts_size, locs_size);
1760   OopMapSet* oop_maps;
1761   int frame_size;
1762   int frame_complete;
1763 
1764   oop_maps = new OopMapSet();
1765   MacroAssembler* masm = new MacroAssembler(&code);
1766 
1767   address start = __ pc();
1768 
1769   frame_size = 2;
1770   __ mov(Rexception_pc, LR);
1771   __ raw_push(FP, LR);
1772 
1773   frame_complete = __ pc() - start;
1774 
1775   // Any extra arguments are already supposed to be R1 and R2
1776   __ mov(R0, Rthread);
1777 
1778   int pc_offset = __ set_last_Java_frame(SP, FP, false, Rtemp);
1779   assert(((__ pc()) - start) == __ offset(), "warning: start differs from code_begin");
1780   __ call(runtime_entry);
1781   if (pc_offset == -1) {
1782     pc_offset = __ offset();
1783   }
1784 
1785   // Generate oop map
1786   OopMap* map =  new OopMap(frame_size*VMRegImpl::slots_per_word, 0);
1787   oop_maps->add_gc_map(pc_offset, map);
1788   __ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call
1789 
1790   __ raw_pop(FP, LR);
1791   __ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
1792 
1793   RuntimeStub* stub = RuntimeStub::new_runtime_stub(name, &code, frame_complete,
1794                                                     frame_size, oop_maps, false);
1795   return stub;
1796 }
1797 
1798 #if INCLUDE_JFR
1799 
1800 // For c2: c_rarg0 is junk, call to runtime to write a checkpoint.
1801 // It returns a jobject handle to the event writer.
1802 // The handle is dereferenced and the return value is the event writer oop.
1803 RuntimeStub* SharedRuntime::generate_jfr_write_checkpoint() {
1804   enum layout {
1805     r1_off,
1806     r2_off,
1807     return_off,
1808     framesize // inclusive of return address
1809   };
1810 
1811   const char* name = SharedRuntime::stub_name(SharedStubId::jfr_write_checkpoint_id);
1812   CodeBuffer code(name, 512, 64);
1813   MacroAssembler* masm = new MacroAssembler(&code);
1814 
1815   address start = __ pc();
1816   __ raw_push(R1, R2, LR);
1817   address the_pc = __ pc();
1818 
1819   int frame_complete = the_pc - start;
1820 
1821   __ set_last_Java_frame(SP, FP, true, Rtemp);
1822   __ mov(c_rarg0, Rthread);
1823   __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::write_checkpoint), c_rarg0);
1824   __ reset_last_Java_frame(Rtemp);
1825 
1826   // R0 is jobject handle result, unpack and process it through a barrier.
1827   __ resolve_global_jobject(R0, Rtemp, R1);
1828 
1829   __ raw_pop(R1, R2, LR);
1830   __ ret();
1831 
1832   OopMapSet* oop_maps = new OopMapSet();
1833   OopMap* map = new OopMap(framesize, 1);
1834   oop_maps->add_gc_map(frame_complete, map);
1835 
1836   RuntimeStub* stub =
1837     RuntimeStub::new_runtime_stub(name,
1838                                   &code,
1839                                   frame_complete,
1840                                   (framesize >> (LogBytesPerWord - LogBytesPerInt)),
1841                                   oop_maps,
1842                                   false);
1843   return stub;
1844 }
1845 
1846 // For c2: call to return a leased buffer.
1847 RuntimeStub* SharedRuntime::generate_jfr_return_lease() {
1848   enum layout {
1849     r1_off,
1850     r2_off,
1851     return_off,
1852     framesize // inclusive of return address
1853   };
1854 
1855   const char* name = SharedRuntime::stub_name(SharedStubId::jfr_return_lease_id);
1856   CodeBuffer code(name, 512, 64);
1857   MacroAssembler* masm = new MacroAssembler(&code);
1858 
1859   address start = __ pc();
1860   __ raw_push(R1, R2, LR);
1861   address the_pc = __ pc();
1862 
1863   int frame_complete = the_pc - start;
1864 
1865   __ set_last_Java_frame(SP, FP, true, Rtemp);
1866   __ mov(c_rarg0, Rthread);
1867   __ call_VM_leaf(CAST_FROM_FN_PTR(address, JfrIntrinsicSupport::return_lease), c_rarg0);
1868   __ reset_last_Java_frame(Rtemp);
1869 
1870   __ raw_pop(R1, R2, LR);
1871   __ ret();
1872 
1873   OopMapSet* oop_maps = new OopMapSet();
1874   OopMap* map = new OopMap(framesize, 1);
1875   oop_maps->add_gc_map(frame_complete, map);
1876 
1877   RuntimeStub* stub =
1878     RuntimeStub::new_runtime_stub(name,
1879                                   &code,
1880                                   frame_complete,
1881                                   (framesize >> (LogBytesPerWord - LogBytesPerInt)),
1882                                   oop_maps,
1883                                   false);
1884   return stub;
1885 }
1886 
1887 #endif // INCLUDE_JFR