1 /*
   2  * Copyright (c) 2008, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.
   8  *
   9  * This code is distributed in the hope that it will be useful, but WITHOUT
  10  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  11  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12  * version 2 for more details (a copy is included in the LICENSE file that
  13  * accompanied this code).
  14  *
  15  * You should have received a copy of the GNU General Public License version
  16  * 2 along with this work; if not, write to the Free Software Foundation,
  17  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  18  *
  19  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  20  * or visit www.oracle.com if you need additional information or have any
  21  * questions.
  22  *
  23  */
  24 
  25 #include "precompiled.hpp"
  26 #include "asm/assembler.inline.hpp"
  27 #include "code/debugInfoRec.hpp"
  28 #include "code/icBuffer.hpp"
  29 #include "code/vtableStubs.hpp"
  30 #include "compiler/oopMap.hpp"
  31 #include "gc/shared/barrierSetAssembler.hpp"
  32 #include "interpreter/interpreter.hpp"
  33 #include "logging/log.hpp"
  34 #include "memory/resourceArea.hpp"
  35 #include "oops/compiledICHolder.hpp"
  36 #include "oops/klass.inline.hpp"
  37 #include "prims/methodHandles.hpp"
  38 #include "runtime/jniHandles.hpp"
  39 #include "runtime/sharedRuntime.hpp"
  40 #include "runtime/safepointMechanism.hpp"
  41 #include "runtime/stubRoutines.hpp"
  42 #include "runtime/vframeArray.hpp"
  43 #include "utilities/align.hpp"
  44 #include "utilities/powerOfTwo.hpp"
  45 #include "vmreg_arm.inline.hpp"
  46 #ifdef COMPILER1
  47 #include "c1/c1_Runtime1.hpp"
  48 #endif
  49 #ifdef COMPILER2
  50 #include "opto/runtime.hpp"
  51 #endif
  52 
  53 #define __ masm->
  54 
  55 class RegisterSaver {
  56 public:
  57 
  58   // Special registers:
  59   //              32-bit ARM     64-bit ARM
  60   //  Rthread:       R10            R28
  61   //  LR:            R14            R30
  62 
  63   // Rthread is callee saved in the C ABI and never changed by compiled code:
  64   // no need to save it.
  65 
  66   // 2 slots for LR: the one at LR_offset and an other one at R14/R30_offset.
  67   // The one at LR_offset is a return address that is needed by stack walking.
  68   // A c2 method uses LR as a standard register so it may be live when we
  69   // branch to the runtime. The slot at R14/R30_offset is for the value of LR
  70   // in case it's live in the method we are coming from.
  71 
  72 
  73   enum RegisterLayout {
  74     fpu_save_size = FloatRegisterImpl::number_of_registers,
  75 #ifndef __SOFTFP__
  76     D0_offset = 0,
  77 #endif
  78     R0_offset = fpu_save_size,
  79     R1_offset,
  80     R2_offset,
  81     R3_offset,
  82     R4_offset,
  83     R5_offset,
  84     R6_offset,
  85 #if (FP_REG_NUM != 7)
  86     // if not saved as FP
  87     R7_offset,
  88 #endif
  89     R8_offset,
  90     R9_offset,
  91 #if (FP_REG_NUM != 11)
  92     // if not saved as FP
  93     R11_offset,
  94 #endif
  95     R12_offset,
  96     R14_offset,
  97     FP_offset,
  98     LR_offset,
  99     reg_save_size,
 100 
 101     Rmethod_offset = R9_offset,
 102     Rtemp_offset = R12_offset,
 103   };
 104 
 105   // all regs but Rthread (R10), FP (R7 or R11), SP and PC
 106   // (altFP_7_11 is the one among R7 and R11 which is not FP)
 107 #define SAVED_BASE_REGS (RegisterSet(R0, R6) | RegisterSet(R8, R9) | RegisterSet(R12) | R14 | altFP_7_11)
 108 
 109 
 110   //  When LR may be live in the nmethod from which we are coming
 111   //  then lr_saved is true, the return address is saved before the
 112   //  call to save_live_register by the caller and LR contains the
 113   //  live value.
 114 
 115   static OopMap* save_live_registers(MacroAssembler* masm,
 116                                      int* total_frame_words,
 117                                      bool lr_saved = false);
 118   static void restore_live_registers(MacroAssembler* masm, bool restore_lr = true);
 119 
 120 };
 121 
 122 
 123 
 124 
 125 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm,
 126                                            int* total_frame_words,
 127                                            bool lr_saved) {
 128   *total_frame_words = reg_save_size;
 129 
 130   OopMapSet *oop_maps = new OopMapSet();
 131   OopMap* map = new OopMap(VMRegImpl::slots_per_word * (*total_frame_words), 0);
 132 
 133   if (lr_saved) {
 134     __ push(RegisterSet(FP));
 135   } else {
 136     __ push(RegisterSet(FP) | RegisterSet(LR));
 137   }
 138   __ push(SAVED_BASE_REGS);
 139   if (HaveVFP) {
 140     if (VM_Version::has_vfp3_32()) {
 141       __ fpush(FloatRegisterSet(D16, 16));
 142     } else {
 143       if (FloatRegisterImpl::number_of_registers > 32) {
 144         assert(FloatRegisterImpl::number_of_registers == 64, "nb fp registers should be 64");
 145         __ sub(SP, SP, 32 * wordSize);
 146       }
 147     }
 148     __ fpush(FloatRegisterSet(D0, 16));
 149   } else {
 150     __ sub(SP, SP, fpu_save_size * wordSize);
 151   }
 152 
 153   int i;
 154   int j=0;
 155   for (i = R0_offset; i <= R9_offset; i++) {
 156     if (j == FP_REG_NUM) {
 157       // skip the FP register, managed below.
 158       j++;
 159     }
 160     map->set_callee_saved(VMRegImpl::stack2reg(i), as_Register(j)->as_VMReg());
 161     j++;
 162   }
 163   assert(j == R10->encoding(), "must be");
 164 #if (FP_REG_NUM != 11)
 165   // add R11, if not managed as FP
 166   map->set_callee_saved(VMRegImpl::stack2reg(R11_offset), R11->as_VMReg());
 167 #endif
 168   map->set_callee_saved(VMRegImpl::stack2reg(R12_offset), R12->as_VMReg());
 169   map->set_callee_saved(VMRegImpl::stack2reg(R14_offset), R14->as_VMReg());
 170   if (HaveVFP) {
 171     for (i = 0; i < (VM_Version::has_vfp3_32() ? 64 : 32); i+=2) {
 172       map->set_callee_saved(VMRegImpl::stack2reg(i), as_FloatRegister(i)->as_VMReg());
 173       map->set_callee_saved(VMRegImpl::stack2reg(i + 1), as_FloatRegister(i)->as_VMReg()->next());
 174     }
 175   }
 176 
 177   return map;
 178 }
 179 
 180 void RegisterSaver::restore_live_registers(MacroAssembler* masm, bool restore_lr) {
 181   if (HaveVFP) {
 182     __ fpop(FloatRegisterSet(D0, 16));
 183     if (VM_Version::has_vfp3_32()) {
 184       __ fpop(FloatRegisterSet(D16, 16));
 185     } else {
 186       if (FloatRegisterImpl::number_of_registers > 32) {
 187         assert(FloatRegisterImpl::number_of_registers == 64, "nb fp registers should be 64");
 188         __ add(SP, SP, 32 * wordSize);
 189       }
 190     }
 191   } else {
 192     __ add(SP, SP, fpu_save_size * wordSize);
 193   }
 194   __ pop(SAVED_BASE_REGS);
 195   if (restore_lr) {
 196     __ pop(RegisterSet(FP) | RegisterSet(LR));
 197   } else {
 198     __ pop(RegisterSet(FP));
 199   }
 200 }
 201 
 202 
 203 static void push_result_registers(MacroAssembler* masm, BasicType ret_type) {
 204 #ifdef __ABI_HARD__
 205   if (ret_type == T_DOUBLE || ret_type == T_FLOAT) {
 206     __ sub(SP, SP, 8);
 207     __ fstd(D0, Address(SP));
 208     return;
 209   }
 210 #endif // __ABI_HARD__
 211   __ raw_push(R0, R1);
 212 }
 213 
 214 static void pop_result_registers(MacroAssembler* masm, BasicType ret_type) {
 215 #ifdef __ABI_HARD__
 216   if (ret_type == T_DOUBLE || ret_type == T_FLOAT) {
 217     __ fldd(D0, Address(SP));
 218     __ add(SP, SP, 8);
 219     return;
 220   }
 221 #endif // __ABI_HARD__
 222   __ raw_pop(R0, R1);
 223 }
 224 
 225 static void push_param_registers(MacroAssembler* masm, int fp_regs_in_arguments) {
 226   // R1-R3 arguments need to be saved, but we push 4 registers for 8-byte alignment
 227   __ push(RegisterSet(R0, R3));
 228 
 229   // preserve arguments
 230   // Likely not needed as the locking code won't probably modify volatile FP registers,
 231   // but there is no way to guarantee that
 232   if (fp_regs_in_arguments) {
 233     // convert fp_regs_in_arguments to a number of double registers
 234     int double_regs_num = (fp_regs_in_arguments + 1) >> 1;
 235     __ fpush_hardfp(FloatRegisterSet(D0, double_regs_num));
 236   }
 237 }
 238 
 239 static void pop_param_registers(MacroAssembler* masm, int fp_regs_in_arguments) {
 240   if (fp_regs_in_arguments) {
 241     int double_regs_num = (fp_regs_in_arguments + 1) >> 1;
 242     __ fpop_hardfp(FloatRegisterSet(D0, double_regs_num));
 243   }
 244   __ pop(RegisterSet(R0, R3));
 245 }
 246 
 247 
 248 
 249 // Is vector's size (in bytes) bigger than a size saved by default?
 250 // All vector registers are saved by default on ARM.
 251 bool SharedRuntime::is_wide_vector(int size) {
 252   return false;
 253 }
 254 
 255 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
 256                                         VMRegPair *regs,
 257                                         VMRegPair *regs2,
 258                                         int total_args_passed) {
 259   assert(regs2 == nullptr, "not needed on arm");
 260 
 261   int slot = 0;
 262   int ireg = 0;
 263 #ifdef __ABI_HARD__
 264   int fp_slot = 0;
 265   int single_fpr_slot = 0;
 266 #endif // __ABI_HARD__
 267   for (int i = 0; i < total_args_passed; i++) {
 268     switch (sig_bt[i]) {
 269     case T_SHORT:
 270     case T_CHAR:
 271     case T_BYTE:
 272     case T_BOOLEAN:
 273     case T_INT:
 274     case T_ARRAY:
 275     case T_OBJECT:
 276     case T_ADDRESS:
 277     case T_METADATA:
 278 #ifndef __ABI_HARD__
 279     case T_FLOAT:
 280 #endif // !__ABI_HARD__
 281       if (ireg < 4) {
 282         Register r = as_Register(ireg);
 283         regs[i].set1(r->as_VMReg());
 284         ireg++;
 285       } else {
 286         regs[i].set1(VMRegImpl::stack2reg(slot));
 287         slot++;
 288       }
 289       break;
 290     case T_LONG:
 291 #ifndef __ABI_HARD__
 292     case T_DOUBLE:
 293 #endif // !__ABI_HARD__
 294       assert((i + 1) < total_args_passed && sig_bt[i+1] == T_VOID, "missing Half" );
 295       if (ireg <= 2) {
 296 #if (ALIGN_WIDE_ARGUMENTS == 1)
 297         if(ireg & 1) ireg++;  // Aligned location required
 298 #endif
 299         Register r1 = as_Register(ireg);
 300         Register r2 = as_Register(ireg + 1);
 301         regs[i].set_pair(r2->as_VMReg(), r1->as_VMReg());
 302         ireg += 2;
 303 #if (ALIGN_WIDE_ARGUMENTS == 0)
 304       } else if (ireg == 3) {
 305         // uses R3 + one stack slot
 306         Register r = as_Register(ireg);
 307         regs[i].set_pair(VMRegImpl::stack2reg(slot), r->as_VMReg());
 308         ireg += 1;
 309         slot += 1;
 310 #endif
 311       } else {
 312         if (slot & 1) slot++; // Aligned location required
 313         regs[i].set_pair(VMRegImpl::stack2reg(slot+1), VMRegImpl::stack2reg(slot));
 314         slot += 2;
 315         ireg = 4;
 316       }
 317       break;
 318     case T_VOID:
 319       regs[i].set_bad();
 320       break;
 321 #ifdef __ABI_HARD__
 322     case T_FLOAT:
 323       if ((fp_slot < 16)||(single_fpr_slot & 1)) {
 324         if ((single_fpr_slot & 1) == 0) {
 325           single_fpr_slot = fp_slot;
 326           fp_slot += 2;
 327         }
 328         FloatRegister r = as_FloatRegister(single_fpr_slot);
 329         single_fpr_slot++;
 330         regs[i].set1(r->as_VMReg());
 331       } else {
 332         regs[i].set1(VMRegImpl::stack2reg(slot));
 333         slot++;
 334       }
 335       break;
 336     case T_DOUBLE:
 337       assert(ALIGN_WIDE_ARGUMENTS == 1, "ABI_HARD not supported with unaligned wide arguments");
 338       if (fp_slot <= 14) {
 339         FloatRegister r1 = as_FloatRegister(fp_slot);
 340         FloatRegister r2 = as_FloatRegister(fp_slot+1);
 341         regs[i].set_pair(r2->as_VMReg(), r1->as_VMReg());
 342         fp_slot += 2;
 343       } else {
 344         if(slot & 1) slot++;
 345         regs[i].set_pair(VMRegImpl::stack2reg(slot+1), VMRegImpl::stack2reg(slot));
 346         slot += 2;
 347         single_fpr_slot = 16;
 348       }
 349       break;
 350 #endif // __ABI_HARD__
 351     default:
 352       ShouldNotReachHere();
 353     }
 354   }
 355   return slot;
 356 }
 357 
 358 int SharedRuntime::vector_calling_convention(VMRegPair *regs,
 359                                              uint num_bits,
 360                                              uint total_args_passed) {
 361   Unimplemented();
 362   return 0;
 363 }
 364 
 365 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
 366                                            VMRegPair *regs,
 367                                            int total_args_passed) {
 368 #ifdef __SOFTFP__
 369   // soft float is the same as the C calling convention.
 370   return c_calling_convention(sig_bt, regs, nullptr, total_args_passed);
 371 #endif // __SOFTFP__
 372   int slot = 0;
 373   int ireg = 0;
 374   int freg = 0;
 375   int single_fpr = 0;
 376 
 377   for (int i = 0; i < total_args_passed; i++) {
 378     switch (sig_bt[i]) {
 379     case T_SHORT:
 380     case T_CHAR:
 381     case T_BYTE:
 382     case T_BOOLEAN:
 383     case T_INT:
 384     case T_ARRAY:
 385     case T_OBJECT:
 386     case T_ADDRESS:
 387       if (ireg < 4) {
 388         Register r = as_Register(ireg++);
 389         regs[i].set1(r->as_VMReg());
 390       } else {
 391         regs[i].set1(VMRegImpl::stack2reg(slot++));
 392       }
 393       break;
 394     case T_FLOAT:
 395       // C2 utilizes S14/S15 for mem-mem moves
 396       if ((freg < 16 COMPILER2_PRESENT(-2)) || (single_fpr & 1)) {
 397         if ((single_fpr & 1) == 0) {
 398           single_fpr = freg;
 399           freg += 2;
 400         }
 401         FloatRegister r = as_FloatRegister(single_fpr++);
 402         regs[i].set1(r->as_VMReg());
 403       } else {
 404         regs[i].set1(VMRegImpl::stack2reg(slot++));
 405       }
 406       break;
 407     case T_DOUBLE:
 408       // C2 utilizes S14/S15 for mem-mem moves
 409       if (freg <= 14 COMPILER2_PRESENT(-2)) {
 410         FloatRegister r1 = as_FloatRegister(freg);
 411         FloatRegister r2 = as_FloatRegister(freg + 1);
 412         regs[i].set_pair(r2->as_VMReg(), r1->as_VMReg());
 413         freg += 2;
 414       } else {
 415         // Keep internally the aligned calling convention,
 416         // ignoring ALIGN_WIDE_ARGUMENTS
 417         if (slot & 1) slot++;
 418         regs[i].set_pair(VMRegImpl::stack2reg(slot + 1), VMRegImpl::stack2reg(slot));
 419         slot += 2;
 420         single_fpr = 16;
 421       }
 422       break;
 423     case T_LONG:
 424       // Keep internally the aligned calling convention,
 425       // ignoring ALIGN_WIDE_ARGUMENTS
 426       if (ireg <= 2) {
 427         if (ireg & 1) ireg++;
 428         Register r1 = as_Register(ireg);
 429         Register r2 = as_Register(ireg + 1);
 430         regs[i].set_pair(r2->as_VMReg(), r1->as_VMReg());
 431         ireg += 2;
 432       } else {
 433         if (slot & 1) slot++;
 434         regs[i].set_pair(VMRegImpl::stack2reg(slot + 1), VMRegImpl::stack2reg(slot));
 435         slot += 2;
 436         ireg = 4;
 437       }
 438       break;
 439     case T_VOID:
 440       regs[i].set_bad();
 441       break;
 442     default:
 443       ShouldNotReachHere();
 444     }
 445   }
 446 
 447   return slot;
 448 }
 449 
 450 static void patch_callers_callsite(MacroAssembler *masm) {
 451   Label skip;
 452 
 453   __ ldr(Rtemp, Address(Rmethod, Method::code_offset()));
 454   __ cbz(Rtemp, skip);
 455 
 456   // Pushing an even number of registers for stack alignment.
 457   // Selecting R9, which had to be saved anyway for some platforms.
 458   __ push(RegisterSet(R0, R3) | R9 | LR);
 459   __ fpush_hardfp(FloatRegisterSet(D0, 8));
 460 
 461   __ mov(R0, Rmethod);
 462   __ mov(R1, LR);
 463   __ call(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite));
 464 
 465   __ fpop_hardfp(FloatRegisterSet(D0, 8));
 466   __ pop(RegisterSet(R0, R3) | R9 | LR);
 467 
 468   __ bind(skip);
 469 }
 470 
 471 void SharedRuntime::gen_i2c_adapter(MacroAssembler *masm,
 472                                     int total_args_passed, int comp_args_on_stack,
 473                                     const BasicType *sig_bt, const VMRegPair *regs) {
 474   // TODO: ARM - May be can use ldm to load arguments
 475   const Register tmp = Rtemp; // avoid erasing R5_mh
 476 
 477   // Next assert may not be needed but safer. Extra analysis required
 478   // if this there is not enough free registers and we need to use R5 here.
 479   assert_different_registers(tmp, R5_mh);
 480 
 481   // 6243940 We might end up in handle_wrong_method if
 482   // the callee is deoptimized as we race thru here. If that
 483   // happens we don't want to take a safepoint because the
 484   // caller frame will look interpreted and arguments are now
 485   // "compiled" so it is much better to make this transition
 486   // invisible to the stack walking code. Unfortunately if
 487   // we try and find the callee by normal means a safepoint
 488   // is possible. So we stash the desired callee in the thread
 489   // and the vm will find there should this case occur.
 490   Address callee_target_addr(Rthread, JavaThread::callee_target_offset());
 491   __ str(Rmethod, callee_target_addr);
 492 
 493 
 494   assert_different_registers(tmp, R0, R1, R2, R3, Rsender_sp, Rmethod);
 495 
 496   const Register initial_sp = Rmethod; // temporarily scratched
 497 
 498   // Old code was modifying R4 but this looks unsafe (particularly with JSR292)
 499   assert_different_registers(tmp, R0, R1, R2, R3, Rsender_sp, initial_sp);
 500 
 501   __ mov(initial_sp, SP);
 502 
 503   if (comp_args_on_stack) {
 504     __ sub_slow(SP, SP, comp_args_on_stack * VMRegImpl::stack_slot_size);
 505   }
 506   __ bic(SP, SP, StackAlignmentInBytes - 1);
 507 
 508   for (int i = 0; i < total_args_passed; i++) {
 509     if (sig_bt[i] == T_VOID) {
 510       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 511       continue;
 512     }
 513     assert(!regs[i].second()->is_valid() || regs[i].first()->next() == regs[i].second(), "must be ordered");
 514     int arg_offset = Interpreter::expr_offset_in_bytes(total_args_passed - 1 - i);
 515 
 516     VMReg r_1 = regs[i].first();
 517     VMReg r_2 = regs[i].second();
 518     if (r_1->is_stack()) {
 519       int stack_offset = r_1->reg2stack() * VMRegImpl::stack_slot_size;
 520       if (!r_2->is_valid()) {
 521         __ ldr(tmp, Address(initial_sp, arg_offset));
 522         __ str(tmp, Address(SP, stack_offset));
 523       } else {
 524         __ ldr(tmp, Address(initial_sp, arg_offset - Interpreter::stackElementSize));
 525         __ str(tmp, Address(SP, stack_offset));
 526         __ ldr(tmp, Address(initial_sp, arg_offset));
 527         __ str(tmp, Address(SP, stack_offset + wordSize));
 528       }
 529     } else if (r_1->is_Register()) {
 530       if (!r_2->is_valid()) {
 531         __ ldr(r_1->as_Register(), Address(initial_sp, arg_offset));
 532       } else {
 533         __ ldr(r_1->as_Register(), Address(initial_sp, arg_offset - Interpreter::stackElementSize));
 534         __ ldr(r_2->as_Register(), Address(initial_sp, arg_offset));
 535       }
 536     } else if (r_1->is_FloatRegister()) {
 537 #ifdef __SOFTFP__
 538       ShouldNotReachHere();
 539 #endif // __SOFTFP__
 540       if (!r_2->is_valid()) {
 541         __ flds(r_1->as_FloatRegister(), Address(initial_sp, arg_offset));
 542       } else {
 543         __ fldd(r_1->as_FloatRegister(), Address(initial_sp, arg_offset - Interpreter::stackElementSize));
 544       }
 545     } else {
 546       assert(!r_1->is_valid() && !r_2->is_valid(), "must be");
 547     }
 548   }
 549 
 550   // restore Rmethod (scratched for initial_sp)
 551   __ ldr(Rmethod, callee_target_addr);
 552   __ ldr(PC, Address(Rmethod, Method::from_compiled_offset()));
 553 
 554 }
 555 
 556 static void gen_c2i_adapter(MacroAssembler *masm,
 557                             int total_args_passed,  int comp_args_on_stack,
 558                             const BasicType *sig_bt, const VMRegPair *regs,
 559                             Label& skip_fixup) {
 560   // TODO: ARM - May be can use stm to deoptimize arguments
 561   const Register tmp = Rtemp;
 562 
 563   patch_callers_callsite(masm);
 564   __ bind(skip_fixup);
 565 
 566   __ mov(Rsender_sp, SP); // not yet saved
 567 
 568 
 569   int extraspace = total_args_passed * Interpreter::stackElementSize;
 570   if (extraspace) {
 571     __ sub_slow(SP, SP, extraspace);
 572   }
 573 
 574   for (int i = 0; i < total_args_passed; i++) {
 575     if (sig_bt[i] == T_VOID) {
 576       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 577       continue;
 578     }
 579     int stack_offset = (total_args_passed - 1 - i) * Interpreter::stackElementSize;
 580 
 581     VMReg r_1 = regs[i].first();
 582     VMReg r_2 = regs[i].second();
 583     if (r_1->is_stack()) {
 584       int arg_offset = r_1->reg2stack() * VMRegImpl::stack_slot_size + extraspace;
 585       if (!r_2->is_valid()) {
 586         __ ldr(tmp, Address(SP, arg_offset));
 587         __ str(tmp, Address(SP, stack_offset));
 588       } else {
 589         __ ldr(tmp, Address(SP, arg_offset));
 590         __ str(tmp, Address(SP, stack_offset - Interpreter::stackElementSize));
 591         __ ldr(tmp, Address(SP, arg_offset + wordSize));
 592         __ str(tmp, Address(SP, stack_offset));
 593       }
 594     } else if (r_1->is_Register()) {
 595       if (!r_2->is_valid()) {
 596         __ str(r_1->as_Register(), Address(SP, stack_offset));
 597       } else {
 598         __ str(r_1->as_Register(), Address(SP, stack_offset - Interpreter::stackElementSize));
 599         __ str(r_2->as_Register(), Address(SP, stack_offset));
 600       }
 601     } else if (r_1->is_FloatRegister()) {
 602 #ifdef __SOFTFP__
 603       ShouldNotReachHere();
 604 #endif // __SOFTFP__
 605       if (!r_2->is_valid()) {
 606         __ fsts(r_1->as_FloatRegister(), Address(SP, stack_offset));
 607       } else {
 608         __ fstd(r_1->as_FloatRegister(), Address(SP, stack_offset - Interpreter::stackElementSize));
 609       }
 610     } else {
 611       assert(!r_1->is_valid() && !r_2->is_valid(), "must be");
 612     }
 613   }
 614 
 615   __ ldr(PC, Address(Rmethod, Method::interpreter_entry_offset()));
 616 
 617 }
 618 
 619 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
 620                                                             int total_args_passed,
 621                                                             int comp_args_on_stack,
 622                                                             const BasicType *sig_bt,
 623                                                             const VMRegPair *regs,
 624                                                             AdapterFingerPrint* fingerprint) {
 625   address i2c_entry = __ pc();
 626   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 627 
 628   address c2i_unverified_entry = __ pc();
 629   Label skip_fixup;
 630   const Register receiver       = R0;
 631   const Register holder_klass   = Rtemp; // XXX should be OK for C2 but not 100% sure
 632   const Register receiver_klass = R4;
 633 
 634   __ load_klass(receiver_klass, receiver);
 635   __ ldr(holder_klass, Address(Ricklass, CompiledICHolder::holder_klass_offset()));
 636   __ ldr(Rmethod, Address(Ricklass, CompiledICHolder::holder_metadata_offset()));
 637   __ cmp(receiver_klass, holder_klass);
 638 
 639   __ ldr(Rtemp, Address(Rmethod, Method::code_offset()), eq);
 640   __ cmp(Rtemp, 0, eq);
 641   __ b(skip_fixup, eq);
 642   __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, noreg, ne);
 643 
 644   address c2i_entry = __ pc();
 645   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
 646 
 647   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry);
 648 }
 649 
 650 
 651 static int reg2offset_in(VMReg r) {
 652   // Account for saved FP and LR
 653   return r->reg2stack() * VMRegImpl::stack_slot_size + 2*wordSize;
 654 }
 655 
 656 static int reg2offset_out(VMReg r) {
 657   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 658 }
 659 
 660 
 661 static void verify_oop_args(MacroAssembler* masm,
 662                             const methodHandle& method,
 663                             const BasicType* sig_bt,
 664                             const VMRegPair* regs) {
 665   Register temp_reg = Rmethod;  // not part of any compiled calling seq
 666   if (VerifyOops) {
 667     for (int i = 0; i < method->size_of_parameters(); i++) {
 668       if (sig_bt[i] == T_OBJECT || sig_bt[i] == T_ARRAY) {
 669         VMReg r = regs[i].first();
 670         assert(r->is_valid(), "bad oop arg");
 671         if (r->is_stack()) {
 672           __ ldr(temp_reg, Address(SP, r->reg2stack() * VMRegImpl::stack_slot_size));
 673           __ verify_oop(temp_reg);
 674         } else {
 675           __ verify_oop(r->as_Register());
 676         }
 677       }
 678     }
 679   }
 680 }
 681 
 682 static void gen_special_dispatch(MacroAssembler* masm,
 683                                  const methodHandle& method,
 684                                  const BasicType* sig_bt,
 685                                  const VMRegPair* regs) {
 686   verify_oop_args(masm, method, sig_bt, regs);
 687   vmIntrinsics::ID iid = method->intrinsic_id();
 688 
 689   // Now write the args into the outgoing interpreter space
 690   bool     has_receiver   = false;
 691   Register receiver_reg   = noreg;
 692   int      member_arg_pos = -1;
 693   Register member_reg     = noreg;
 694   int      ref_kind       = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
 695   if (ref_kind != 0) {
 696     member_arg_pos = method->size_of_parameters() - 1;  // trailing MemberName argument
 697     member_reg = Rmethod;  // known to be free at this point
 698     has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
 699   } else if (iid == vmIntrinsics::_invokeBasic) {
 700     has_receiver = true;
 701   } else {
 702     fatal("unexpected intrinsic id %d", vmIntrinsics::as_int(iid));
 703   }
 704 
 705   if (member_reg != noreg) {
 706     // Load the member_arg into register, if necessary.
 707     SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
 708     VMReg r = regs[member_arg_pos].first();
 709     if (r->is_stack()) {
 710       __ ldr(member_reg, Address(SP, r->reg2stack() * VMRegImpl::stack_slot_size));
 711     } else {
 712       // no data motion is needed
 713       member_reg = r->as_Register();
 714     }
 715   }
 716 
 717   if (has_receiver) {
 718     // Make sure the receiver is loaded into a register.
 719     assert(method->size_of_parameters() > 0, "oob");
 720     assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
 721     VMReg r = regs[0].first();
 722     assert(r->is_valid(), "bad receiver arg");
 723     if (r->is_stack()) {
 724       // Porting note:  This assumes that compiled calling conventions always
 725       // pass the receiver oop in a register.  If this is not true on some
 726       // platform, pick a temp and load the receiver from stack.
 727       assert(false, "receiver always in a register");
 728       receiver_reg = j_rarg0;  // known to be free at this point
 729       __ ldr(receiver_reg, Address(SP, r->reg2stack() * VMRegImpl::stack_slot_size));
 730     } else {
 731       // no data motion is needed
 732       receiver_reg = r->as_Register();
 733     }
 734   }
 735 
 736   // Figure out which address we are really jumping to:
 737   MethodHandles::generate_method_handle_dispatch(masm, iid,
 738                                                  receiver_reg, member_reg, /*for_compiler_entry:*/ true);
 739 }
 740 
 741 // ---------------------------------------------------------------------------
 742 // Generate a native wrapper for a given method.  The method takes arguments
 743 // in the Java compiled code convention, marshals them to the native
 744 // convention (handlizes oops, etc), transitions to native, makes the call,
 745 // returns to java state (possibly blocking), unhandlizes any result and
 746 // returns.
 747 nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
 748                                                 const methodHandle& method,
 749                                                 int compile_id,
 750                                                 BasicType* in_sig_bt,
 751                                                 VMRegPair* in_regs,
 752                                                 BasicType ret_type) {
 753   if (method->is_method_handle_intrinsic()) {
 754     vmIntrinsics::ID iid = method->intrinsic_id();
 755     intptr_t start = (intptr_t)__ pc();
 756     int vep_offset = ((intptr_t)__ pc()) - start;
 757     gen_special_dispatch(masm,
 758                          method,
 759                          in_sig_bt,
 760                          in_regs);
 761     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
 762     __ flush();
 763     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
 764     return nmethod::new_native_nmethod(method,
 765                                        compile_id,
 766                                        masm->code(),
 767                                        vep_offset,
 768                                        frame_complete,
 769                                        stack_slots / VMRegImpl::slots_per_word,
 770                                        in_ByteSize(-1),
 771                                        in_ByteSize(-1),
 772                                        (OopMapSet*)nullptr);
 773   }
 774   // Arguments for JNI method include JNIEnv and Class if static
 775 
 776   // Usage of Rtemp should be OK since scratched by native call
 777 
 778   bool method_is_static = method->is_static();
 779 
 780   const int total_in_args = method->size_of_parameters();
 781   int total_c_args = total_in_args + (method_is_static ? 2 : 1);
 782 
 783   BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
 784   VMRegPair* out_regs   = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
 785 
 786   int argc = 0;
 787   out_sig_bt[argc++] = T_ADDRESS;
 788   if (method_is_static) {
 789     out_sig_bt[argc++] = T_OBJECT;
 790   }
 791 
 792   int i;
 793   for (i = 0; i < total_in_args; i++) {
 794     out_sig_bt[argc++] = in_sig_bt[i];
 795   }
 796 
 797   int out_arg_slots = c_calling_convention(out_sig_bt, out_regs, nullptr, total_c_args);
 798   int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots;
 799   // Since object arguments need to be wrapped, we must preserve space
 800   // for those object arguments which come in registers (GPR_PARAMS maximum)
 801   // plus one more slot for Klass handle (for static methods)
 802   int oop_handle_offset = stack_slots;
 803   stack_slots += (GPR_PARAMS + 1) * VMRegImpl::slots_per_word;
 804 
 805   // Plus a lock if needed
 806   int lock_slot_offset = 0;
 807   if (method->is_synchronized()) {
 808     lock_slot_offset = stack_slots;
 809     assert(sizeof(BasicLock) == wordSize, "adjust this code");
 810     stack_slots += VMRegImpl::slots_per_word;
 811   }
 812 
 813   // Space to save return address and FP
 814   stack_slots += 2 * VMRegImpl::slots_per_word;
 815 
 816   // Calculate the final stack size taking account of alignment
 817   stack_slots = align_up(stack_slots, StackAlignmentInBytes / VMRegImpl::stack_slot_size);
 818   int stack_size = stack_slots * VMRegImpl::stack_slot_size;
 819   int lock_slot_fp_offset = stack_size - 2 * wordSize -
 820     lock_slot_offset * VMRegImpl::stack_slot_size;
 821 
 822   // Unverified entry point
 823   address start = __ pc();
 824 
 825   // Inline cache check, same as in C1_MacroAssembler::inline_cache_check()
 826   const Register receiver = R0; // see receiverOpr()
 827   __ load_klass(Rtemp, receiver);
 828   __ cmp(Rtemp, Ricklass);
 829   Label verified;
 830 
 831   __ b(verified, eq); // jump over alignment no-ops too
 832   __ jump(SharedRuntime::get_ic_miss_stub(), relocInfo::runtime_call_type, Rtemp);
 833   __ align(CodeEntryAlignment);
 834 
 835   // Verified entry point
 836   __ bind(verified);
 837   int vep_offset = __ pc() - start;
 838 
 839 
 840   if ((InlineObjectHash && method->intrinsic_id() == vmIntrinsics::_hashCode) || (method->intrinsic_id() == vmIntrinsics::_identityHashCode)) {
 841     // Object.hashCode, System.identityHashCode can pull the hashCode from the header word
 842     // instead of doing a full VM transition once it's been computed.
 843     Label slow_case;
 844     const Register obj_reg = R0;
 845 
 846     // Unlike for Object.hashCode, System.identityHashCode is static method and
 847     // gets object as argument instead of the receiver.
 848     if (method->intrinsic_id() == vmIntrinsics::_identityHashCode) {
 849       assert(method->is_static(), "method should be static");
 850       // return 0 for null reference input, return val = R0 = obj_reg = 0
 851       __ cmp(obj_reg, 0);
 852       __ bx(LR, eq);
 853     }
 854 
 855     __ ldr(Rtemp, Address(obj_reg, oopDesc::mark_offset_in_bytes()));
 856 
 857     assert(markWord::unlocked_value == 1, "adjust this code");
 858     __ tbz(Rtemp, exact_log2(markWord::unlocked_value), slow_case);
 859 
 860     __ bics(Rtemp, Rtemp, ~markWord::hash_mask_in_place);
 861     __ mov(R0, AsmOperand(Rtemp, lsr, markWord::hash_shift), ne);
 862     __ bx(LR, ne);
 863 
 864     __ bind(slow_case);
 865   }
 866 
 867   // Bang stack pages
 868   __ arm_stack_overflow_check(stack_size, Rtemp);
 869 
 870   // Setup frame linkage
 871   __ raw_push(FP, LR);
 872   __ mov(FP, SP);
 873   __ sub_slow(SP, SP, stack_size - 2*wordSize);
 874 
 875   BarrierSetAssembler* bs = BarrierSet::barrier_set()->barrier_set_assembler();
 876   assert(bs != nullptr, "Sanity");
 877   bs->nmethod_entry_barrier(masm);
 878 
 879   int frame_complete = __ pc() - start;
 880 
 881   OopMapSet* oop_maps = new OopMapSet();
 882   OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
 883   const int extra_args = method_is_static ? 2 : 1;
 884   int receiver_offset = -1;
 885   int fp_regs_in_arguments = 0;
 886 
 887   for (i = total_in_args; --i >= 0; ) {
 888     switch (in_sig_bt[i]) {
 889     case T_ARRAY:
 890     case T_OBJECT: {
 891       VMReg src = in_regs[i].first();
 892       VMReg dst = out_regs[i + extra_args].first();
 893       if (src->is_stack()) {
 894         assert(dst->is_stack(), "must be");
 895         assert(i != 0, "Incoming receiver is always in a register");
 896         __ ldr(Rtemp, Address(FP, reg2offset_in(src)));
 897         __ cmp(Rtemp, 0);
 898         __ add(Rtemp, FP, reg2offset_in(src), ne);
 899         __ str(Rtemp, Address(SP, reg2offset_out(dst)));
 900         int offset_in_older_frame = src->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 901         map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + stack_slots));
 902       } else {
 903         int offset = oop_handle_offset * VMRegImpl::stack_slot_size;
 904         __ str(src->as_Register(), Address(SP, offset));
 905         map->set_oop(VMRegImpl::stack2reg(oop_handle_offset));
 906         if ((i == 0) && (!method_is_static)) {
 907           receiver_offset = offset;
 908         }
 909         oop_handle_offset += VMRegImpl::slots_per_word;
 910 
 911         if (dst->is_stack()) {
 912           __ movs(Rtemp, src->as_Register());
 913           __ add(Rtemp, SP, offset, ne);
 914           __ str(Rtemp, Address(SP, reg2offset_out(dst)));
 915         } else {
 916           __ movs(dst->as_Register(), src->as_Register());
 917           __ add(dst->as_Register(), SP, offset, ne);
 918         }
 919       }
 920     }
 921 
 922     case T_VOID:
 923       break;
 924 
 925 
 926 #ifdef __SOFTFP__
 927     case T_DOUBLE:
 928 #endif
 929     case T_LONG: {
 930       VMReg src_1 = in_regs[i].first();
 931       VMReg src_2 = in_regs[i].second();
 932       VMReg dst_1 = out_regs[i + extra_args].first();
 933       VMReg dst_2 = out_regs[i + extra_args].second();
 934 #if (ALIGN_WIDE_ARGUMENTS == 0)
 935       // C convention can mix a register and a stack slot for a
 936       // 64-bits native argument.
 937 
 938       // Note: following code should work independently of whether
 939       // the Java calling convention follows C convention or whether
 940       // it aligns 64-bit values.
 941       if (dst_2->is_Register()) {
 942         if (src_1->as_Register() != dst_1->as_Register()) {
 943           assert(src_1->as_Register() != dst_2->as_Register() &&
 944                  src_2->as_Register() != dst_2->as_Register(), "must be");
 945           __ mov(dst_2->as_Register(), src_2->as_Register());
 946           __ mov(dst_1->as_Register(), src_1->as_Register());
 947         } else {
 948           assert(src_2->as_Register() == dst_2->as_Register(), "must be");
 949         }
 950       } else if (src_2->is_Register()) {
 951         if (dst_1->is_Register()) {
 952           // dst mixes a register and a stack slot
 953           assert(dst_2->is_stack() && src_1->is_Register() && src_2->is_Register(), "must be");
 954           assert(src_1->as_Register() != dst_1->as_Register(), "must be");
 955           __ str(src_2->as_Register(), Address(SP, reg2offset_out(dst_2)));
 956           __ mov(dst_1->as_Register(), src_1->as_Register());
 957         } else {
 958           // registers to stack slots
 959           assert(dst_2->is_stack() && src_1->is_Register() && src_2->is_Register(), "must be");
 960           __ str(src_1->as_Register(), Address(SP, reg2offset_out(dst_1)));
 961           __ str(src_2->as_Register(), Address(SP, reg2offset_out(dst_2)));
 962         }
 963       } else if (src_1->is_Register()) {
 964         if (dst_1->is_Register()) {
 965           // src and dst must be R3 + stack slot
 966           assert(dst_1->as_Register() == src_1->as_Register(), "must be");
 967           __ ldr(Rtemp,    Address(FP, reg2offset_in(src_2)));
 968           __ str(Rtemp,    Address(SP, reg2offset_out(dst_2)));
 969         } else {
 970           // <R3,stack> -> <stack,stack>
 971           assert(dst_2->is_stack() && src_2->is_stack(), "must be");
 972           __ ldr(LR, Address(FP, reg2offset_in(src_2)));
 973           __ str(src_1->as_Register(), Address(SP, reg2offset_out(dst_1)));
 974           __ str(LR, Address(SP, reg2offset_out(dst_2)));
 975         }
 976       } else {
 977         assert(src_2->is_stack() && dst_1->is_stack() && dst_2->is_stack(), "must be");
 978         __ ldr(Rtemp, Address(FP, reg2offset_in(src_1)));
 979         __ ldr(LR,    Address(FP, reg2offset_in(src_2)));
 980         __ str(Rtemp, Address(SP, reg2offset_out(dst_1)));
 981         __ str(LR,    Address(SP, reg2offset_out(dst_2)));
 982       }
 983 #else // ALIGN_WIDE_ARGUMENTS
 984       if (src_1->is_stack()) {
 985         assert(src_2->is_stack() && dst_1->is_stack() && dst_2->is_stack(), "must be");
 986         __ ldr(Rtemp, Address(FP, reg2offset_in(src_1)));
 987         __ ldr(LR,    Address(FP, reg2offset_in(src_2)));
 988         __ str(Rtemp, Address(SP, reg2offset_out(dst_1)));
 989         __ str(LR,    Address(SP, reg2offset_out(dst_2)));
 990       } else if (dst_1->is_stack()) {
 991         assert(dst_2->is_stack() && src_1->is_Register() && src_2->is_Register(), "must be");
 992         __ str(src_1->as_Register(), Address(SP, reg2offset_out(dst_1)));
 993         __ str(src_2->as_Register(), Address(SP, reg2offset_out(dst_2)));
 994       } else if (src_1->as_Register() == dst_1->as_Register()) {
 995         assert(src_2->as_Register() == dst_2->as_Register(), "must be");
 996       } else {
 997         assert(src_1->as_Register() != dst_2->as_Register() &&
 998                src_2->as_Register() != dst_2->as_Register(), "must be");
 999         __ mov(dst_2->as_Register(), src_2->as_Register());
1000         __ mov(dst_1->as_Register(), src_1->as_Register());
1001       }
1002 #endif // ALIGN_WIDE_ARGUMENTS
1003       break;
1004     }
1005 
1006 #if (!defined __SOFTFP__ && !defined __ABI_HARD__)
1007     case T_FLOAT: {
1008       VMReg src = in_regs[i].first();
1009       VMReg dst = out_regs[i + extra_args].first();
1010       if (src->is_stack()) {
1011         assert(dst->is_stack(), "must be");
1012         __ ldr(Rtemp, Address(FP, reg2offset_in(src)));
1013         __ str(Rtemp, Address(SP, reg2offset_out(dst)));
1014       } else if (dst->is_stack()) {
1015         __ fsts(src->as_FloatRegister(), Address(SP, reg2offset_out(dst)));
1016       } else {
1017         assert(src->is_FloatRegister() && dst->is_Register(), "must be");
1018         __ fmrs(dst->as_Register(), src->as_FloatRegister());
1019       }
1020       break;
1021     }
1022 
1023     case T_DOUBLE: {
1024       VMReg src_1 = in_regs[i].first();
1025       VMReg src_2 = in_regs[i].second();
1026       VMReg dst_1 = out_regs[i + extra_args].first();
1027       VMReg dst_2 = out_regs[i + extra_args].second();
1028       if (src_1->is_stack()) {
1029         assert(src_2->is_stack() && dst_1->is_stack() && dst_2->is_stack(), "must be");
1030         __ ldr(Rtemp, Address(FP, reg2offset_in(src_1)));
1031         __ ldr(LR,    Address(FP, reg2offset_in(src_2)));
1032         __ str(Rtemp, Address(SP, reg2offset_out(dst_1)));
1033         __ str(LR,    Address(SP, reg2offset_out(dst_2)));
1034       } else if (dst_1->is_stack()) {
1035         assert(dst_2->is_stack() && src_1->is_FloatRegister(), "must be");
1036         __ fstd(src_1->as_FloatRegister(), Address(SP, reg2offset_out(dst_1)));
1037 #if (ALIGN_WIDE_ARGUMENTS == 0)
1038       } else if (dst_2->is_stack()) {
1039         assert(! src_2->is_stack(), "must be"); // assuming internal java convention is aligned
1040         // double register must go into R3 + one stack slot
1041         __ fmrrd(dst_1->as_Register(), Rtemp, src_1->as_FloatRegister());
1042         __ str(Rtemp, Address(SP, reg2offset_out(dst_2)));
1043 #endif
1044       } else {
1045         assert(src_1->is_FloatRegister() && dst_1->is_Register() && dst_2->is_Register(), "must be");
1046         __ fmrrd(dst_1->as_Register(), dst_2->as_Register(), src_1->as_FloatRegister());
1047       }
1048       break;
1049     }
1050 #endif // __SOFTFP__
1051 
1052 #ifdef __ABI_HARD__
1053     case T_FLOAT: {
1054       VMReg src = in_regs[i].first();
1055       VMReg dst = out_regs[i + extra_args].first();
1056       if (src->is_stack()) {
1057         if (dst->is_stack()) {
1058           __ ldr(Rtemp, Address(FP, reg2offset_in(src)));
1059           __ str(Rtemp, Address(SP, reg2offset_out(dst)));
1060         } else {
1061           // C2 Java calling convention does not populate S14 and S15, therefore
1062           // those need to be loaded from stack here
1063           __ flds(dst->as_FloatRegister(), Address(FP, reg2offset_in(src)));
1064           fp_regs_in_arguments++;
1065         }
1066       } else {
1067         assert(src->is_FloatRegister(), "must be");
1068         fp_regs_in_arguments++;
1069       }
1070       break;
1071     }
1072     case T_DOUBLE: {
1073       VMReg src_1 = in_regs[i].first();
1074       VMReg src_2 = in_regs[i].second();
1075       VMReg dst_1 = out_regs[i + extra_args].first();
1076       VMReg dst_2 = out_regs[i + extra_args].second();
1077       if (src_1->is_stack()) {
1078         if (dst_1->is_stack()) {
1079           assert(dst_2->is_stack(), "must be");
1080           __ ldr(Rtemp, Address(FP, reg2offset_in(src_1)));
1081           __ ldr(LR,    Address(FP, reg2offset_in(src_2)));
1082           __ str(Rtemp, Address(SP, reg2offset_out(dst_1)));
1083           __ str(LR,    Address(SP, reg2offset_out(dst_2)));
1084         } else {
1085           // C2 Java calling convention does not populate S14 and S15, therefore
1086           // those need to be loaded from stack here
1087           __ fldd(dst_1->as_FloatRegister(), Address(FP, reg2offset_in(src_1)));
1088           fp_regs_in_arguments += 2;
1089         }
1090       } else {
1091         assert(src_1->is_FloatRegister() && src_2->is_FloatRegister(), "must be");
1092         fp_regs_in_arguments += 2;
1093       }
1094       break;
1095     }
1096 #endif // __ABI_HARD__
1097 
1098     default: {
1099       assert(in_sig_bt[i] != T_ADDRESS, "found T_ADDRESS in java args");
1100       VMReg src = in_regs[i].first();
1101       VMReg dst = out_regs[i + extra_args].first();
1102       if (src->is_stack()) {
1103         assert(dst->is_stack(), "must be");
1104         __ ldr(Rtemp, Address(FP, reg2offset_in(src)));
1105         __ str(Rtemp, Address(SP, reg2offset_out(dst)));
1106       } else if (dst->is_stack()) {
1107         __ str(src->as_Register(), Address(SP, reg2offset_out(dst)));
1108       } else {
1109         assert(src->is_Register() && dst->is_Register(), "must be");
1110         __ mov(dst->as_Register(), src->as_Register());
1111       }
1112     }
1113     }
1114   }
1115 
1116   // Get Klass mirror
1117   int klass_offset = -1;
1118   if (method_is_static) {
1119     klass_offset = oop_handle_offset * VMRegImpl::stack_slot_size;
1120     __ mov_oop(Rtemp, JNIHandles::make_local(method->method_holder()->java_mirror()));
1121     __ add(c_rarg1, SP, klass_offset);
1122     __ str(Rtemp, Address(SP, klass_offset));
1123     map->set_oop(VMRegImpl::stack2reg(oop_handle_offset));
1124   }
1125 
1126   // the PC offset given to add_gc_map must match the PC saved in set_last_Java_frame
1127   int pc_offset = __ set_last_Java_frame(SP, FP, true, Rtemp);
1128   assert(((__ pc()) - start) == __ offset(), "warning: start differs from code_begin");
1129   oop_maps->add_gc_map(pc_offset, map);
1130 
1131   // Order last_Java_pc store with the thread state transition (to _thread_in_native)
1132   __ membar(MacroAssembler::StoreStore, Rtemp);
1133 
1134   // RedefineClasses() tracing support for obsolete method entry
1135   if (log_is_enabled(Trace, redefine, class, obsolete)) {
1136     __ save_caller_save_registers();
1137     __ mov(R0, Rthread);
1138     __ mov_metadata(R1, method());
1139     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry), R0, R1);
1140     __ restore_caller_save_registers();
1141   }
1142 
1143   const Register sync_handle = R5;
1144   const Register sync_obj    = R6;
1145   const Register disp_hdr    = altFP_7_11;
1146   const Register tmp         = R8;
1147 
1148   Label slow_lock, lock_done, fast_lock;
1149   if (method->is_synchronized()) {
1150     // The first argument is a handle to sync object (a class or an instance)
1151     __ ldr(sync_obj, Address(R1));
1152     // Remember the handle for the unlocking code
1153     __ mov(sync_handle, R1);
1154 
1155     if (LockingMode == LM_LIGHTWEIGHT) {
1156       log_trace(fastlock)("SharedRuntime lock fast");
1157       __ lightweight_lock(sync_obj /* object */, disp_hdr /* t1 */, tmp /* t2 */, Rtemp /* t3 */,
1158                           0x7 /* savemask */, slow_lock);
1159       // Fall through to lock_done
1160     } else if (LockingMode == LM_LEGACY) {
1161       const Register mark = tmp;
1162       // On MP platforms the next load could return a 'stale' value if the memory location has been modified by another thread.
1163       // That would be acceptable as either CAS or slow case path is taken in that case
1164 
1165       __ ldr(mark, Address(sync_obj, oopDesc::mark_offset_in_bytes()));
1166       __ sub(disp_hdr, FP, lock_slot_fp_offset);
1167       __ tst(mark, markWord::unlocked_value);
1168       __ b(fast_lock, ne);
1169 
1170       // Check for recursive lock
1171       // See comments in InterpreterMacroAssembler::lock_object for
1172       // explanations on the fast recursive locking check.
1173       // Check independently the low bits and the distance to SP
1174       // -1- test low 2 bits
1175       __ movs(Rtemp, AsmOperand(mark, lsl, 30));
1176       // -2- test (hdr - SP) if the low two bits are 0
1177       __ sub(Rtemp, mark, SP, eq);
1178       __ movs(Rtemp, AsmOperand(Rtemp, lsr, exact_log2(os::vm_page_size())), eq);
1179       // If still 'eq' then recursive locking OK
1180       // set to zero if recursive lock, set to non zero otherwise (see discussion in JDK-8267042)
1181       __ str(Rtemp, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()));
1182       __ b(lock_done, eq);
1183       __ b(slow_lock);
1184 
1185       __ bind(fast_lock);
1186       __ str(mark, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()));
1187 
1188       __ cas_for_lock_acquire(mark, disp_hdr, sync_obj, Rtemp, slow_lock);
1189     }
1190     __ bind(lock_done);
1191   }
1192 
1193   // Get JNIEnv*
1194   __ add(c_rarg0, Rthread, in_bytes(JavaThread::jni_environment_offset()));
1195 
1196   // Perform thread state transition
1197   __ mov(Rtemp, _thread_in_native);
1198   __ str(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1199 
1200   // Finally, call the native method
1201   __ call(method->native_function());
1202 
1203   // Set FPSCR/FPCR to a known state
1204   if (AlwaysRestoreFPU) {
1205     __ restore_default_fp_mode();
1206   }
1207 
1208   // Ensure a Boolean result is mapped to 0..1
1209   if (ret_type == T_BOOLEAN) {
1210     __ c2bool(R0);
1211   }
1212 
1213   // Do a safepoint check while thread is in transition state
1214   Label call_safepoint_runtime, return_to_java;
1215   __ mov(Rtemp, _thread_in_native_trans);
1216   __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1217 
1218   // make sure the store is observed before reading the SafepointSynchronize state and further mem refs
1219   if (!UseSystemMemoryBarrier) {
1220     __ membar(MacroAssembler::Membar_mask_bits(MacroAssembler::StoreLoad | MacroAssembler::StoreStore), Rtemp);
1221   }
1222 
1223   __ safepoint_poll(R2, call_safepoint_runtime);
1224   __ ldr_u32(R3, Address(Rthread, JavaThread::suspend_flags_offset()));
1225   __ cmp(R3, 0);
1226   __ b(call_safepoint_runtime, ne);
1227 
1228   __ bind(return_to_java);
1229 
1230   // Perform thread state transition and reguard stack yellow pages if needed
1231   Label reguard, reguard_done;
1232   __ mov(Rtemp, _thread_in_Java);
1233   __ ldr_s32(R2, Address(Rthread, JavaThread::stack_guard_state_offset()));
1234   __ str_32(Rtemp, Address(Rthread, JavaThread::thread_state_offset()));
1235 
1236   __ cmp(R2, StackOverflow::stack_guard_yellow_reserved_disabled);
1237   __ b(reguard, eq);
1238   __ bind(reguard_done);
1239 
1240   Label slow_unlock, unlock_done;
1241   if (method->is_synchronized()) {
1242     if (LockingMode == LM_LIGHTWEIGHT) {
1243       log_trace(fastlock)("SharedRuntime unlock fast");
1244       __ lightweight_unlock(sync_obj, R2 /* t1 */, tmp /* t2 */, Rtemp /* t3 */,
1245                             7 /* savemask */, slow_unlock);
1246       // Fall through
1247     } else if (LockingMode == LM_LEGACY) {
1248       // See C1_MacroAssembler::unlock_object() for more comments
1249       __ ldr(sync_obj, Address(sync_handle));
1250 
1251       // See C1_MacroAssembler::unlock_object() for more comments
1252       __ ldr(R2, Address(disp_hdr, BasicLock::displaced_header_offset_in_bytes()));
1253       __ cbz(R2, unlock_done);
1254 
1255       __ cas_for_lock_release(disp_hdr, R2, sync_obj, Rtemp, slow_unlock);
1256     }
1257     __ bind(unlock_done);
1258   }
1259 
1260   // Set last java frame and handle block to zero
1261   __ ldr(LR, Address(Rthread, JavaThread::active_handles_offset()));
1262   __ reset_last_Java_frame(Rtemp); // sets Rtemp to 0 on 32-bit ARM
1263 
1264   __ str_32(Rtemp, Address(LR, JNIHandleBlock::top_offset()));
1265   if (CheckJNICalls) {
1266     __ str(__ zero_register(Rtemp), Address(Rthread, JavaThread::pending_jni_exception_check_fn_offset()));
1267   }
1268 
1269   // Unbox oop result, e.g. JNIHandles::resolve value in R0.
1270   if (ret_type == T_OBJECT || ret_type == T_ARRAY) {
1271     __ resolve_jobject(R0,      // value
1272                        Rtemp,   // tmp1
1273                        R1_tmp); // tmp2
1274   }
1275 
1276   // Any exception pending?
1277   __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
1278   __ mov(SP, FP);
1279 
1280   __ cmp(Rtemp, 0);
1281   // Pop the frame and return if no exception pending
1282   __ pop(RegisterSet(FP) | RegisterSet(PC), eq);
1283   // Pop the frame and forward the exception. Rexception_pc contains return address.
1284   __ ldr(FP, Address(SP, wordSize, post_indexed), ne);
1285   __ ldr(Rexception_pc, Address(SP, wordSize, post_indexed), ne);
1286   __ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
1287 
1288   // Safepoint operation and/or pending suspend request is in progress.
1289   // Save the return values and call the runtime function by hand.
1290   __ bind(call_safepoint_runtime);
1291   push_result_registers(masm, ret_type);
1292   __ mov(R0, Rthread);
1293   __ call(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans));
1294   pop_result_registers(masm, ret_type);
1295   __ b(return_to_java);
1296 
1297   // Reguard stack pages. Save native results around a call to C runtime.
1298   __ bind(reguard);
1299   push_result_registers(masm, ret_type);
1300   __ call(CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
1301   pop_result_registers(masm, ret_type);
1302   __ b(reguard_done);
1303 
1304   if (method->is_synchronized()) {
1305     // Locking slow case
1306     __ bind(slow_lock);
1307 
1308     push_param_registers(masm, fp_regs_in_arguments);
1309 
1310     // last_Java_frame is already set, so do call_VM manually; no exception can occur
1311     __ mov(R0, sync_obj);
1312     __ mov(R1, disp_hdr);
1313     __ mov(R2, Rthread);
1314     __ call(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C));
1315 
1316     pop_param_registers(masm, fp_regs_in_arguments);
1317 
1318     __ b(lock_done);
1319 
1320     // Unlocking slow case
1321     __ bind(slow_unlock);
1322 
1323     push_result_registers(masm, ret_type);
1324 
1325     // Clear pending exception before reentering VM.
1326     // Can store the oop in register since it is a leaf call.
1327     assert_different_registers(Rtmp_save1, sync_obj, disp_hdr);
1328     __ ldr(Rtmp_save1, Address(Rthread, Thread::pending_exception_offset()));
1329     Register zero = __ zero_register(Rtemp);
1330     __ str(zero, Address(Rthread, Thread::pending_exception_offset()));
1331     __ mov(R0, sync_obj);
1332     __ mov(R1, disp_hdr);
1333     __ mov(R2, Rthread);
1334     __ call(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C));
1335     __ str(Rtmp_save1, Address(Rthread, Thread::pending_exception_offset()));
1336 
1337     pop_result_registers(masm, ret_type);
1338 
1339     __ b(unlock_done);
1340   }
1341 
1342   __ flush();
1343   return nmethod::new_native_nmethod(method,
1344                                      compile_id,
1345                                      masm->code(),
1346                                      vep_offset,
1347                                      frame_complete,
1348                                      stack_slots / VMRegImpl::slots_per_word,
1349                                      in_ByteSize(method_is_static ? klass_offset : receiver_offset),
1350                                      in_ByteSize(lock_slot_offset * VMRegImpl::stack_slot_size),
1351                                      oop_maps);
1352 }
1353 
1354 // this function returns the adjust size (in number of words) to a c2i adapter
1355 // activation for use during deoptimization
1356 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals) {
1357   int extra_locals_size = (callee_locals - callee_parameters) * Interpreter::stackElementWords;
1358   return extra_locals_size;
1359 }
1360 
1361 
1362 // Number of stack slots between incoming argument block and the start of
1363 // a new frame.  The PROLOG must add this many slots to the stack.  The
1364 // EPILOG must remove this many slots.
1365 // FP + LR
1366 uint SharedRuntime::in_preserve_stack_slots() {
1367   return 2 * VMRegImpl::slots_per_word;
1368 }
1369 
1370 uint SharedRuntime::out_preserve_stack_slots() {
1371   return 0;
1372 }
1373 
1374 //------------------------------generate_deopt_blob----------------------------
1375 void SharedRuntime::generate_deopt_blob() {
1376   ResourceMark rm;
1377   CodeBuffer buffer("deopt_blob", 1024, 1024);
1378   int frame_size_in_words;
1379   OopMapSet* oop_maps;
1380   int reexecute_offset;
1381   int exception_in_tls_offset;
1382   int exception_offset;
1383 
1384   MacroAssembler* masm = new MacroAssembler(&buffer);
1385   Label cont;
1386   const Register Rkind   = R9; // caller-saved
1387   const Register Rublock = R6;
1388   const Register Rsender = altFP_7_11;
1389   assert_different_registers(Rkind, Rublock, Rsender, Rexception_obj, Rexception_pc, R0, R1, R2, R3, R8, Rtemp);
1390 
1391   address start = __ pc();
1392 
1393   oop_maps = new OopMapSet();
1394   // LR saved by caller (can be live in c2 method)
1395 
1396   // A deopt is a case where LR may be live in the c2 nmethod. So it's
1397   // not possible to call the deopt blob from the nmethod and pass the
1398   // address of the deopt handler of the nmethod in LR. What happens
1399   // now is that the caller of the deopt blob pushes the current
1400   // address so the deopt blob doesn't have to do it. This way LR can
1401   // be preserved, contains the live value from the nmethod and is
1402   // saved at R14/R30_offset here.
1403   OopMap* map = RegisterSaver::save_live_registers(masm, &frame_size_in_words, true);
1404   __ mov(Rkind, Deoptimization::Unpack_deopt);
1405   __ b(cont);
1406 
1407   exception_offset = __ pc() - start;
1408 
1409   // Transfer Rexception_obj & Rexception_pc in TLS and fall thru to the
1410   // exception_in_tls_offset entry point.
1411   __ str(Rexception_obj, Address(Rthread, JavaThread::exception_oop_offset()));
1412   __ str(Rexception_pc, Address(Rthread, JavaThread::exception_pc_offset()));
1413   // Force return value to null to avoid confusing the escape analysis
1414   // logic. Everything is dead here anyway.
1415   __ mov(R0, 0);
1416 
1417   exception_in_tls_offset = __ pc() - start;
1418 
1419   // Exception data is in JavaThread structure
1420   // Patch the return address of the current frame
1421   __ ldr(LR, Address(Rthread, JavaThread::exception_pc_offset()));
1422   (void) RegisterSaver::save_live_registers(masm, &frame_size_in_words);
1423   {
1424     const Register Rzero = __ zero_register(Rtemp); // XXX should be OK for C2 but not 100% sure
1425     __ str(Rzero, Address(Rthread, JavaThread::exception_pc_offset()));
1426   }
1427   __ mov(Rkind, Deoptimization::Unpack_exception);
1428   __ b(cont);
1429 
1430   reexecute_offset = __ pc() - start;
1431 
1432   (void) RegisterSaver::save_live_registers(masm, &frame_size_in_words);
1433   __ mov(Rkind, Deoptimization::Unpack_reexecute);
1434 
1435   // Calculate UnrollBlock and save the result in Rublock
1436   __ bind(cont);
1437   __ mov(R0, Rthread);
1438   __ mov(R1, Rkind);
1439 
1440   int pc_offset = __ set_last_Java_frame(SP, FP, false, Rtemp); // note: FP may not need to be saved (not on x86)
1441   assert(((__ pc()) - start) == __ offset(), "warning: start differs from code_begin");
1442   __ call(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info));
1443   if (pc_offset == -1) {
1444     pc_offset = __ offset();
1445   }
1446   oop_maps->add_gc_map(pc_offset, map);
1447   __ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call
1448 
1449   __ mov(Rublock, R0);
1450 
1451   // Reload Rkind from the UnrollBlock (might have changed)
1452   __ ldr_s32(Rkind, Address(Rublock, Deoptimization::UnrollBlock::unpack_kind_offset()));
1453   Label noException;
1454   __ cmp_32(Rkind, Deoptimization::Unpack_exception);   // Was exception pending?
1455   __ b(noException, ne);
1456   // handle exception case
1457 #ifdef ASSERT
1458   // assert that exception_pc is zero in tls
1459   { Label L;
1460     __ ldr(Rexception_pc, Address(Rthread, JavaThread::exception_pc_offset()));
1461     __ cbz(Rexception_pc, L);
1462     __ stop("exception pc should be null");
1463     __ bind(L);
1464   }
1465 #endif
1466   __ ldr(Rexception_obj, Address(Rthread, JavaThread::exception_oop_offset()));
1467   __ verify_oop(Rexception_obj);
1468   {
1469     const Register Rzero = __ zero_register(Rtemp);
1470     __ str(Rzero, Address(Rthread, JavaThread::exception_oop_offset()));
1471   }
1472 
1473   __ bind(noException);
1474 
1475   // This frame is going away.  Fetch return value, so we can move it to
1476   // a new frame.
1477   __ ldr(R0, Address(SP, RegisterSaver::R0_offset * wordSize));
1478   __ ldr(R1, Address(SP, RegisterSaver::R1_offset * wordSize));
1479 #ifndef __SOFTFP__
1480   __ ldr_double(D0, Address(SP, RegisterSaver::D0_offset * wordSize));
1481 #endif
1482   // pop frame
1483   __ add(SP, SP, RegisterSaver::reg_save_size * wordSize);
1484 
1485   // Set initial stack state before pushing interpreter frames
1486   __ ldr_s32(Rtemp, Address(Rublock, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset()));
1487   __ ldr(R2, Address(Rublock, Deoptimization::UnrollBlock::frame_pcs_offset()));
1488   __ ldr(R3, Address(Rublock, Deoptimization::UnrollBlock::frame_sizes_offset()));
1489 
1490   __ add(SP, SP, Rtemp);
1491 
1492 #ifdef ASSERT
1493   // Compilers generate code that bang the stack by as much as the
1494   // interpreter would need. So this stack banging should never
1495   // trigger a fault. Verify that it does not on non product builds.
1496   // See if it is enough stack to push deoptimized frames.
1497   //
1498   // The compiled method that we are deoptimizing was popped from the stack.
1499   // If the stack bang results in a stack overflow, we don't return to the
1500   // method that is being deoptimized. The stack overflow exception is
1501   // propagated to the caller of the deoptimized method. Need to get the pc
1502   // from the caller in LR and restore FP.
1503   __ ldr(LR, Address(R2, 0));
1504   __ ldr(FP, Address(Rublock, Deoptimization::UnrollBlock::initial_info_offset()));
1505   __ ldr_s32(R8, Address(Rublock, Deoptimization::UnrollBlock::total_frame_sizes_offset()));
1506   __ arm_stack_overflow_check(R8, Rtemp);
1507 #endif
1508   __ ldr_s32(R8, Address(Rublock, Deoptimization::UnrollBlock::number_of_frames_offset()));
1509 
1510   // Pick up the initial fp we should save
1511   // XXX Note: was ldr(FP, Address(FP));
1512 
1513   // The compiler no longer uses FP as a frame pointer for the
1514   // compiled code. It can be used by the allocator in C2 or to
1515   // memorize the original SP for JSR292 call sites.
1516 
1517   // Hence, ldr(FP, Address(FP)) is probably not correct. For x86,
1518   // Deoptimization::fetch_unroll_info computes the right FP value and
1519   // stores it in Rublock.initial_info. This has been activated for ARM.
1520   __ ldr(FP, Address(Rublock, Deoptimization::UnrollBlock::initial_info_offset()));
1521 
1522   __ ldr_s32(Rtemp, Address(Rublock, Deoptimization::UnrollBlock::caller_adjustment_offset()));
1523   __ mov(Rsender, SP);
1524   __ sub(SP, SP, Rtemp);
1525 
1526   // Push interpreter frames in a loop
1527   Label loop;
1528   __ bind(loop);
1529   __ ldr(LR, Address(R2, wordSize, post_indexed));         // load frame pc
1530   __ ldr(Rtemp, Address(R3, wordSize, post_indexed));      // load frame size
1531 
1532   __ raw_push(FP, LR);                                     // create new frame
1533   __ mov(FP, SP);
1534   __ sub(Rtemp, Rtemp, 2*wordSize);
1535 
1536   __ sub(SP, SP, Rtemp);
1537 
1538   __ str(Rsender, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize));
1539   __ mov(LR, 0);
1540   __ str(LR, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1541 
1542   __ subs(R8, R8, 1);                               // decrement counter
1543   __ mov(Rsender, SP);
1544   __ b(loop, ne);
1545 
1546   // Re-push self-frame
1547   __ ldr(LR, Address(R2));
1548   __ raw_push(FP, LR);
1549   __ mov(FP, SP);
1550   __ sub(SP, SP, (frame_size_in_words - 2) * wordSize);
1551 
1552   // Restore frame locals after moving the frame
1553   __ str(R0, Address(SP, RegisterSaver::R0_offset * wordSize));
1554   __ str(R1, Address(SP, RegisterSaver::R1_offset * wordSize));
1555 
1556 #ifndef __SOFTFP__
1557   __ str_double(D0, Address(SP, RegisterSaver::D0_offset * wordSize));
1558 #endif // !__SOFTFP__
1559 
1560 #ifdef ASSERT
1561   // Reload Rkind from the UnrollBlock and check that it was not overwritten (Rkind is not callee-saved)
1562   { Label L;
1563     __ ldr_s32(Rtemp, Address(Rublock, Deoptimization::UnrollBlock::unpack_kind_offset()));
1564     __ cmp_32(Rkind, Rtemp);
1565     __ b(L, eq);
1566     __ stop("Rkind was overwritten");
1567     __ bind(L);
1568   }
1569 #endif
1570 
1571   // Call unpack_frames with proper arguments
1572   __ mov(R0, Rthread);
1573   __ mov(R1, Rkind);
1574 
1575   pc_offset = __ set_last_Java_frame(SP, FP, true, Rtemp);
1576   assert(((__ pc()) - start) == __ offset(), "warning: start differs from code_begin");
1577   __ call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames));
1578   if (pc_offset == -1) {
1579     pc_offset = __ offset();
1580   }
1581   oop_maps->add_gc_map(pc_offset, new OopMap(frame_size_in_words * VMRegImpl::slots_per_word, 0));
1582   __ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call
1583 
1584   // Collect return values, pop self-frame and jump to interpreter
1585   __ ldr(R0, Address(SP, RegisterSaver::R0_offset * wordSize));
1586   __ ldr(R1, Address(SP, RegisterSaver::R1_offset * wordSize));
1587   // Interpreter floats controlled by __SOFTFP__, but compiler
1588   // float return value registers controlled by __ABI_HARD__
1589   // This matters for vfp-sflt builds.
1590 #ifndef __SOFTFP__
1591   // Interpreter hard float
1592 #ifdef __ABI_HARD__
1593   // Compiler float return value in FP registers
1594   __ ldr_double(D0, Address(SP, RegisterSaver::D0_offset * wordSize));
1595 #else
1596   // Compiler float return value in integer registers,
1597   // copy to D0 for interpreter (S0 <-- R0)
1598   __ fmdrr(D0_tos, R0, R1);
1599 #endif
1600 #endif // !__SOFTFP__
1601   __ mov(SP, FP);
1602 
1603   __ pop(RegisterSet(FP) | RegisterSet(PC));
1604 
1605   __ flush();
1606 
1607   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset,
1608                                            reexecute_offset, frame_size_in_words);
1609   _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
1610 }
1611 
1612 #ifdef COMPILER2
1613 
1614 //------------------------------generate_uncommon_trap_blob--------------------
1615 // Ought to generate an ideal graph & compile, but here's some ASM
1616 // instead.
1617 void SharedRuntime::generate_uncommon_trap_blob() {
1618   // allocate space for the code
1619   ResourceMark rm;
1620 
1621   // setup code generation tools
1622 #ifdef _LP64
1623   CodeBuffer buffer("uncommon_trap_blob", 2700, 512);
1624 #else
1625   // Measured 8/7/03 at 660 in 32bit debug build
1626   CodeBuffer buffer("uncommon_trap_blob", 2000, 512);
1627 #endif
1628   // bypassed when code generation useless
1629   MacroAssembler* masm               = new MacroAssembler(&buffer);
1630   const Register Rublock = R6;
1631   const Register Rsender = altFP_7_11;
1632   assert_different_registers(Rublock, Rsender, Rexception_obj, R0, R1, R2, R3, R8, Rtemp);
1633 
1634   //
1635   // This is the entry point for all traps the compiler takes when it thinks
1636   // it cannot handle further execution of compilation code. The frame is
1637   // deoptimized in these cases and converted into interpreter frames for
1638   // execution
1639   // The steps taken by this frame are as follows:
1640   //   - push a fake "unpack_frame"
1641   //   - call the C routine Deoptimization::uncommon_trap (this function
1642   //     packs the current compiled frame into vframe arrays and returns
1643   //     information about the number and size of interpreter frames which
1644   //     are equivalent to the frame which is being deoptimized)
1645   //   - deallocate the "unpack_frame"
1646   //   - deallocate the deoptimization frame
1647   //   - in a loop using the information returned in the previous step
1648   //     push interpreter frames;
1649   //   - create a dummy "unpack_frame"
1650   //   - call the C routine: Deoptimization::unpack_frames (this function
1651   //     lays out values on the interpreter frame which was just created)
1652   //   - deallocate the dummy unpack_frame
1653   //   - return to the interpreter entry point
1654   //
1655   //  Refer to the following methods for more information:
1656   //   - Deoptimization::uncommon_trap
1657   //   - Deoptimization::unpack_frame
1658 
1659   // the unloaded class index is in R0 (first parameter to this blob)
1660 
1661   __ raw_push(FP, LR);
1662   __ set_last_Java_frame(SP, FP, false, Rtemp);
1663   __ mov(R2, Deoptimization::Unpack_uncommon_trap);
1664   __ mov(R1, R0);
1665   __ mov(R0, Rthread);
1666   __ call(CAST_FROM_FN_PTR(address, Deoptimization::uncommon_trap));
1667   __ mov(Rublock, R0);
1668   __ reset_last_Java_frame(Rtemp);
1669   __ raw_pop(FP, LR);
1670 
1671 #ifdef ASSERT
1672   { Label L;
1673     __ ldr_s32(Rtemp, Address(Rublock, Deoptimization::UnrollBlock::unpack_kind_offset()));
1674     __ cmp_32(Rtemp, Deoptimization::Unpack_uncommon_trap);
1675     __ b(L, eq);
1676     __ stop("SharedRuntime::generate_uncommon_trap_blob: expected Unpack_uncommon_trap");
1677     __ bind(L);
1678   }
1679 #endif
1680 
1681 
1682   // Set initial stack state before pushing interpreter frames
1683   __ ldr_s32(Rtemp, Address(Rublock, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset()));
1684   __ ldr(R2, Address(Rublock, Deoptimization::UnrollBlock::frame_pcs_offset()));
1685   __ ldr(R3, Address(Rublock, Deoptimization::UnrollBlock::frame_sizes_offset()));
1686 
1687   __ add(SP, SP, Rtemp);
1688 
1689   // See if it is enough stack to push deoptimized frames.
1690 #ifdef ASSERT
1691   // Compilers generate code that bang the stack by as much as the
1692   // interpreter would need. So this stack banging should never
1693   // trigger a fault. Verify that it does not on non product builds.
1694   //
1695   // The compiled method that we are deoptimizing was popped from the stack.
1696   // If the stack bang results in a stack overflow, we don't return to the
1697   // method that is being deoptimized. The stack overflow exception is
1698   // propagated to the caller of the deoptimized method. Need to get the pc
1699   // from the caller in LR and restore FP.
1700   __ ldr(LR, Address(R2, 0));
1701   __ ldr(FP, Address(Rublock, Deoptimization::UnrollBlock::initial_info_offset()));
1702   __ ldr_s32(R8, Address(Rublock, Deoptimization::UnrollBlock::total_frame_sizes_offset()));
1703   __ arm_stack_overflow_check(R8, Rtemp);
1704 #endif
1705   __ ldr_s32(R8, Address(Rublock, Deoptimization::UnrollBlock::number_of_frames_offset()));
1706   __ ldr_s32(Rtemp, Address(Rublock, Deoptimization::UnrollBlock::caller_adjustment_offset()));
1707   __ mov(Rsender, SP);
1708   __ sub(SP, SP, Rtemp);
1709   //  __ ldr(FP, Address(FP));
1710   __ ldr(FP, Address(Rublock, Deoptimization::UnrollBlock::initial_info_offset()));
1711 
1712   // Push interpreter frames in a loop
1713   Label loop;
1714   __ bind(loop);
1715   __ ldr(LR, Address(R2, wordSize, post_indexed));         // load frame pc
1716   __ ldr(Rtemp, Address(R3, wordSize, post_indexed));      // load frame size
1717 
1718   __ raw_push(FP, LR);                                     // create new frame
1719   __ mov(FP, SP);
1720   __ sub(Rtemp, Rtemp, 2*wordSize);
1721 
1722   __ sub(SP, SP, Rtemp);
1723 
1724   __ str(Rsender, Address(FP, frame::interpreter_frame_sender_sp_offset * wordSize));
1725   __ mov(LR, 0);
1726   __ str(LR, Address(FP, frame::interpreter_frame_last_sp_offset * wordSize));
1727   __ subs(R8, R8, 1);                               // decrement counter
1728   __ mov(Rsender, SP);
1729   __ b(loop, ne);
1730 
1731   // Re-push self-frame
1732   __ ldr(LR, Address(R2));
1733   __ raw_push(FP, LR);
1734   __ mov(FP, SP);
1735 
1736   // Call unpack_frames with proper arguments
1737   __ mov(R0, Rthread);
1738   __ mov(R1, Deoptimization::Unpack_uncommon_trap);
1739   __ set_last_Java_frame(SP, FP, true, Rtemp);
1740   __ call_VM_leaf(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames));
1741   //  oop_maps->add_gc_map(__ pc() - start, new OopMap(frame_size_in_words, 0));
1742   __ reset_last_Java_frame(Rtemp);
1743 
1744   __ mov(SP, FP);
1745   __ pop(RegisterSet(FP) | RegisterSet(PC));
1746 
1747   masm->flush();
1748   _uncommon_trap_blob = UncommonTrapBlob::create(&buffer, nullptr, 2 /* LR+FP */);
1749 }
1750 
1751 #endif // COMPILER2
1752 
1753 //------------------------------generate_handler_blob------
1754 //
1755 // Generate a special Compile2Runtime blob that saves all registers,
1756 // setup oopmap, and calls safepoint code to stop the compiled code for
1757 // a safepoint.
1758 //
1759 SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_type) {
1760   assert(StubRoutines::forward_exception_entry() != nullptr, "must be generated before");
1761 
1762   ResourceMark rm;
1763   CodeBuffer buffer("handler_blob", 256, 256);
1764   int frame_size_words;
1765   OopMapSet* oop_maps;
1766 
1767   bool cause_return = (poll_type == POLL_AT_RETURN);
1768 
1769   MacroAssembler* masm = new MacroAssembler(&buffer);
1770   address start = __ pc();
1771   oop_maps = new OopMapSet();
1772 
1773   if (!cause_return) {
1774     __ sub(SP, SP, 4); // make room for LR which may still be live
1775                        // here if we are coming from a c2 method
1776   }
1777 
1778   OopMap* map = RegisterSaver::save_live_registers(masm, &frame_size_words, !cause_return);
1779   if (!cause_return) {
1780     // update saved PC with correct value
1781     // need 2 steps because LR can be live in c2 method
1782     __ ldr(LR, Address(Rthread, JavaThread::saved_exception_pc_offset()));
1783     __ str(LR, Address(SP, RegisterSaver::LR_offset * wordSize));
1784   }
1785 
1786   __ mov(R0, Rthread);
1787   int pc_offset = __ set_last_Java_frame(SP, FP, false, Rtemp); // note: FP may not need to be saved (not on x86)
1788   assert(((__ pc()) - start) == __ offset(), "warning: start differs from code_begin");
1789   __ call(call_ptr);
1790   if (pc_offset == -1) {
1791     pc_offset = __ offset();
1792   }
1793   oop_maps->add_gc_map(pc_offset, map);
1794   __ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call
1795 
1796   if (!cause_return) {
1797     // If our stashed return pc was modified by the runtime we avoid touching it
1798     __ ldr(R3_tmp, Address(Rthread, JavaThread::saved_exception_pc_offset()));
1799     __ ldr(R2_tmp, Address(SP, RegisterSaver::LR_offset * wordSize));
1800     __ cmp(R2_tmp, R3_tmp);
1801     // Adjust return pc forward to step over the safepoint poll instruction
1802     __ add(R2_tmp, R2_tmp, 4, eq);
1803     __ str(R2_tmp, Address(SP, RegisterSaver::LR_offset * wordSize), eq);
1804 
1805     // Check for pending exception
1806     __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
1807     __ cmp(Rtemp, 0);
1808 
1809     RegisterSaver::restore_live_registers(masm, false);
1810     __ pop(PC, eq);
1811     __ pop(Rexception_pc);
1812   } else {
1813     // Check for pending exception
1814     __ ldr(Rtemp, Address(Rthread, Thread::pending_exception_offset()));
1815     __ cmp(Rtemp, 0);
1816 
1817     RegisterSaver::restore_live_registers(masm);
1818     __ bx(LR, eq);
1819     __ mov(Rexception_pc, LR);
1820   }
1821 
1822   __ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
1823 
1824   __ flush();
1825 
1826   return SafepointBlob::create(&buffer, oop_maps, frame_size_words);
1827 }
1828 
1829 RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const char* name) {
1830   assert(StubRoutines::forward_exception_entry() != nullptr, "must be generated before");
1831 
1832   ResourceMark rm;
1833   CodeBuffer buffer(name, 1000, 512);
1834   int frame_size_words;
1835   OopMapSet *oop_maps;
1836   int frame_complete;
1837 
1838   MacroAssembler* masm = new MacroAssembler(&buffer);
1839   Label pending_exception;
1840 
1841   int start = __ offset();
1842 
1843   oop_maps = new OopMapSet();
1844   OopMap* map = RegisterSaver::save_live_registers(masm, &frame_size_words);
1845 
1846   frame_complete = __ offset();
1847 
1848   __ mov(R0, Rthread);
1849 
1850   int pc_offset = __ set_last_Java_frame(SP, FP, false, Rtemp);
1851   assert(start == 0, "warning: start differs from code_begin");
1852   __ call(destination);
1853   if (pc_offset == -1) {
1854     pc_offset = __ offset();
1855   }
1856   oop_maps->add_gc_map(pc_offset, map);
1857   __ reset_last_Java_frame(Rtemp); // Rtemp free since scratched by far call
1858 
1859   __ ldr(R1, Address(Rthread, Thread::pending_exception_offset()));
1860   __ cbnz(R1, pending_exception);
1861 
1862   // Overwrite saved register values
1863 
1864   // Place metadata result of VM call into Rmethod
1865   __ get_vm_result_2(R1, Rtemp);
1866   __ str(R1, Address(SP, RegisterSaver::Rmethod_offset * wordSize));
1867 
1868   // Place target address (VM call result) into Rtemp
1869   __ str(R0, Address(SP, RegisterSaver::Rtemp_offset * wordSize));
1870 
1871   RegisterSaver::restore_live_registers(masm);
1872   __ jump(Rtemp);
1873 
1874   __ bind(pending_exception);
1875 
1876   RegisterSaver::restore_live_registers(masm);
1877   const Register Rzero = __ zero_register(Rtemp);
1878   __ str(Rzero, Address(Rthread, JavaThread::vm_result_2_offset()));
1879   __ mov(Rexception_pc, LR);
1880   __ jump(StubRoutines::forward_exception_entry(), relocInfo::runtime_call_type, Rtemp);
1881 
1882   __ flush();
1883 
1884   return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_words, oop_maps, true);
1885 }