1 /*
   2  * Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2015, Linaro Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/macroAssembler.hpp"
  29 #include "asm/macroAssembler.inline.hpp"
  30 #include "code/debugInfoRec.hpp"
  31 #include "code/icBuffer.hpp"
  32 #include "code/vtableStubs.hpp"
  33 #include "interp_masm_aarch32.hpp"
  34 #include "interpreter/interpreter.hpp"
  35 #include "oops/compiledICHolder.hpp"
  36 #include "prims/jvmtiRedefineClassesTrace.hpp"
  37 #include "runtime/sharedRuntime.hpp"
  38 #include "runtime/vframeArray.hpp"
  39 #include "vmreg_aarch32.inline.hpp"
  40 #include "register_aarch32.hpp"
  41 #include "vm_version_aarch32.hpp"
  42 #ifdef COMPILER1
  43 #include "c1/c1_Runtime1.hpp"
  44 #endif
  45 #ifdef COMPILER2
  46 #include "adfiles/ad_aarch32.hpp"
  47 #include "opto/runtime.hpp"
  48 #endif
  49 
  50 
  51 #define __ masm->
  52 
  53 const int StackAlignmentInSlots = StackAlignmentInBytes / VMRegImpl::stack_slot_size;
  54 
  55 class SimpleRuntimeFrame {
  56 
  57   public:
  58 
  59   // Most of the runtime stubs have this simple frame layout.
  60   // This class exists to make the layout shared in one place.
  61   // Offsets are for compiler stack slots, which are jints.
  62   enum layout {
  63     // The frame sender code expects that rbp will be in the "natural" place and
  64     // will override any oopMap setting for it. We must therefore force the layout
  65     // so that it agrees with the frame sender code.
  66     // we don't expect any arg reg save area so aarch32 asserts that
  67     // frame::arg_reg_save_area_bytes == 0
  68     rbp_off = 0,
  69     rbp_off2,
  70     return_off, return_off2,
  71     framesize
  72   };
  73 };
  74 
  75 // FIXME -- this is used by C1
  76 class RegisterSaver {
  77  public:
  78   static OopMap* save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words);
  79   static void restore_live_registers(MacroAssembler* masm);
  80 
  81   // Capture info about frame layout
  82   enum layout {
  83       fpu_state_off = 0,
  84       fpu_state_end = fpu_state_off+FPUStateSizeInWords-1,
  85       // The frame sender code expects that rfp will be in
  86       // the "natural" place and will override any oopMap
  87       // setting for it. We must therefore force the layout
  88       // so that it agrees with the frame sender code.
  89       //
  90       // FIXME there are extra saved register (from `push_CPU_state`) note that r11 == rfp
  91       r0_off,
  92       r1_off,
  93       r2_off,
  94       r3_off,
  95       r4_off,
  96       r5_off,
  97       r6_off,
  98       r7_off,
  99       r8_off,
 100       r9_off,  rscratch1_off = r9_off,
 101       r10_off, rmethod_off = r10_off,
 102       r11_off,
 103       r12_off,
 104       reg_save_pad, // align area to 8-bytes to simplify stack alignment to 8
 105       rfp_off,
 106       return_off,
 107       reg_save_size,
 108   };
 109 
 110 
 111   // Offsets into the register save area
 112   // Used by deoptimization when it is managing result register
 113   // values on its own
 114 
 115   static int offset_in_bytes(int offset)    { return offset * wordSize; }
 116 
 117 // During deoptimization only the result registers need to be restored,
 118   // all the other values have already been extracted.
 119   static void restore_result_registers(MacroAssembler* masm);
 120 
 121 };
 122 
 123 OopMap* RegisterSaver::save_live_registers(MacroAssembler* masm, int additional_frame_words, int* total_frame_words) {
 124   int frame_size_in_bytes = additional_frame_words*wordSize + reg_save_size*BytesPerInt;
 125   int frame_size_in_slots = frame_size_in_bytes / BytesPerInt;
 126   int additional_frame_slots = additional_frame_words*wordSize / BytesPerInt;
 127   *total_frame_words = frame_size_in_bytes / wordSize;;
 128 
 129   __ enter();
 130   __ push_CPU_state();
 131 
 132   // Set an oopmap for the call site.  This oopmap will map all
 133   // oop-registers and debug-info registers as callee-saved.  This
 134   // will allow deoptimization at this safepoint to find all possible
 135   // debug-info recordings, as well as let GC find all oops.
 136 
 137   OopMapSet *oop_maps = new OopMapSet();
 138   OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
 139 
 140   oop_map->set_callee_saved(VMRegImpl::stack2reg(r0_off + additional_frame_slots), r0->as_VMReg());
 141   oop_map->set_callee_saved(VMRegImpl::stack2reg(r1_off + additional_frame_slots), r1->as_VMReg());
 142   oop_map->set_callee_saved(VMRegImpl::stack2reg(r2_off + additional_frame_slots), r2->as_VMReg());
 143   oop_map->set_callee_saved(VMRegImpl::stack2reg(r3_off + additional_frame_slots), r3->as_VMReg());
 144   oop_map->set_callee_saved(VMRegImpl::stack2reg(r4_off + additional_frame_slots), r4->as_VMReg());
 145   oop_map->set_callee_saved(VMRegImpl::stack2reg(r5_off + additional_frame_slots), r5->as_VMReg());
 146   oop_map->set_callee_saved(VMRegImpl::stack2reg(r6_off + additional_frame_slots), r6->as_VMReg());
 147   oop_map->set_callee_saved(VMRegImpl::stack2reg(r7_off + additional_frame_slots), r7->as_VMReg());
 148   oop_map->set_callee_saved(VMRegImpl::stack2reg(r8_off + additional_frame_slots), r8->as_VMReg());
 149   oop_map->set_callee_saved(VMRegImpl::stack2reg(r10_off + additional_frame_slots), r10->as_VMReg());
 150   // r11 saved in frame header as rfp, not map it here
 151   // r11 & r14 have special meaning (can't hold oop), so not map them
 152   if(hasFPU()) {
 153   for (int i = 0; i < FPUStateSizeInWords; ++i) {
 154     oop_map->set_callee_saved(VMRegImpl::stack2reg(fpu_state_off + i + additional_frame_slots),
 155     as_FloatRegister(i)->as_VMReg());
 156   }
 157   }
 158 
 159   return oop_map;
 160 }
 161 
 162 void RegisterSaver::restore_live_registers(MacroAssembler* masm) {
 163   __ pop_CPU_state();
 164   __ leave();
 165 }
 166 
 167 void RegisterSaver::restore_result_registers(MacroAssembler* masm) {
 168 
 169   // Just restore result register. Only used by deoptimization. By
 170   // now any callee save register that needs to be restored to a c2
 171   // caller of the deoptee has been extracted into the vframeArray
 172   // and will be stuffed into the c2i adapter we create for later
 173   // restoration so only result registers need to be restored here.
 174 
 175 
 176   if(hasFPU()) {
 177   // Restore fp result register
 178   __ vldr_f64(d0, Address(sp, offset_in_bytes(fpu_state_off)));
 179   }
 180 
 181   // Restore integer result register
 182   __ ldr(r0, Address(sp, offset_in_bytes(r0_off)));
 183   __ ldr(r1, Address(sp, offset_in_bytes(r1_off)));
 184 
 185   // Pop all of the register save are off the stack
 186   __ add(sp, sp, reg_save_size * wordSize);
 187 }
 188 
 189 // Is vector's size (in bytes) bigger than a size saved by default?
 190 // 16 bytes XMM registers are saved by default using fxsave/fxrstor instructions.
 191 bool SharedRuntime::is_wide_vector(int size) {
 192   return size > 16;
 193 }
 194 
 195 // This functions returns offset from fp to java arguments on stack.
 196 //
 197 // The java_calling_convention describes stack locations as ideal slots on
 198 // a frame with no abi restrictions. Since we must observe abi restrictions
 199 // (like the placement of the register window) the slots must be biased by
 200 // the following value.
 201 static int reg2offset_in(VMReg r) {
 202   // After stack frame created, fp points to 1 slot after previous sp value.
 203   return (r->reg2stack() + 1) * VMRegImpl::stack_slot_size;
 204 }
 205 
 206 static int reg2offset_out(VMReg r) {
 207   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
 208 }
 209 
 210 // ---------------------------------------------------------------------------
 211 // Read the array of BasicTypes from a signature, and compute where the
 212 // arguments should go.  Values in the VMRegPair regs array refer to 4-byte
 213 // quantities.  Values less than VMRegImpl::stack0 are registers, those above
 214 // refer to 4-byte stack slots.  All stack slots are based off of the stack pointer
 215 // as framesizes are fixed.
 216 // VMRegImpl::stack0 refers to the first slot 0(sp).
 217 // and VMRegImpl::stack0+1 refers to the memory word 4-byes higher.  Register
 218 // up to RegisterImpl::number_of_registers) are the 64-bit
 219 // integer registers.
 220 
 221 // Note: the INPUTS in sig_bt are in units of Java argument words,
 222 // which are 64-bit.  The OUTPUTS are in 32-bit units.
 223 
 224 // The Java calling convention is a "shifted" version of the C ABI.
 225 // By skipping the first C ABI register we can call non-static jni
 226 // methods with small numbers of arguments without having to shuffle
 227 // the arguments at all. Since we control the java ABI we ought to at
 228 // least get some advantage out of it.
 229 
 230 int SharedRuntime::java_calling_convention(const BasicType *sig_bt,
 231                                            VMRegPair *regs,
 232                                            int total_args_passed,
 233                                            int is_outgoing) {
 234 
 235   // Create the mapping between argument positions and
 236   // registers.
 237   static const Register INT_ArgReg[Argument::n_int_register_parameters_j] = {
 238     j_rarg0, j_rarg1, j_rarg2, j_rarg3
 239   };
 240   const int FP_ArgReg_N = 16;
 241   static const FloatRegister FP_ArgReg[] = {
 242     f0, f1, f2, f3,
 243     f4, f5, f6, f7,
 244     f8, f9, f10, f11,
 245     f12, f13, f14, f15,
 246   };
 247 
 248   uint int_args = 0;
 249   uint fp_args = 0;
 250   uint stk_args = 0;
 251 
 252   for (int i = 0; i < total_args_passed; i++) {
 253     switch (sig_bt[i]) {
 254     case T_FLOAT:
 255         if(hasFPU()) {
 256             if (fp_args < FP_ArgReg_N) {
 257               regs[i].set1(FP_ArgReg[fp_args++]->as_VMReg());
 258             } else {
 259               regs[i].set1(VMRegImpl::stack2reg(stk_args));
 260               stk_args += 1;
 261             }
 262             break;
 263         } // fallthough for no-FPU system
 264     case T_BOOLEAN:
 265     case T_CHAR:
 266     case T_BYTE:
 267     case T_SHORT:
 268     case T_INT:
 269     case T_OBJECT:
 270     case T_ARRAY:
 271     case T_ADDRESS:
 272       if (int_args < Argument::n_int_register_parameters_j) {
 273         regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
 274       } else {
 275         regs[i].set1(VMRegImpl::stack2reg(stk_args));
 276         stk_args += 1;
 277       }
 278       break;
 279     case T_VOID:
 280       // halves of T_LONG or T_DOUBLE
 281       assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 282       regs[i].set_bad();
 283       break;
 284     case T_DOUBLE:
 285         if(hasFPU()) {
 286             assert(sig_bt[i + 1] == T_VOID, "expecting half");
 287             fp_args = round_to(fp_args, 2);
 288             if (fp_args < FP_ArgReg_N) {
 289               regs[i].set2(FP_ArgReg[fp_args]->as_VMReg());
 290               fp_args += 2;
 291             } else {
 292               regs[i].set2(VMRegImpl::stack2reg(stk_args));
 293               stk_args += 2;
 294             }
 295             break;
 296         } //fallthough for no-FPU system
 297     case T_LONG:
 298       assert(sig_bt[i + 1] == T_VOID, "expecting half");
 299       if (int_args + 1 < Argument::n_int_register_parameters_j) {
 300         regs[i].set_pair(INT_ArgReg[int_args + 1]->as_VMReg(), INT_ArgReg[int_args]->as_VMReg());
 301         int_args += 2;
 302       } else {
 303         regs[i].set2(VMRegImpl::stack2reg(stk_args));
 304         stk_args += 2;
 305       }
 306       break;
 307     default:
 308       ShouldNotReachHere();
 309       break;
 310     }
 311   }
 312 
 313   return round_to(stk_args, StackAlignmentInBytes/wordSize);
 314 }
 315 
 316 // Patch the callers callsite with entry to compiled code if it exists.
 317 static void patch_callers_callsite(MacroAssembler *masm) {
 318   Label L;
 319   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 320   __ cbz(rscratch1, L);
 321 
 322   __ enter();
 323   __ push_CPU_state();
 324 
 325   // VM needs caller's callsite
 326   // VM needs target method
 327   // This needs to be a long call since we will relocate this adapter to
 328   // the codeBuffer and it may not reach
 329 
 330 #ifndef PRODUCT
 331   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
 332 #endif
 333 
 334   __ mov(c_rarg0, rmethod);
 335   __ mov(c_rarg1, lr);
 336   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, SharedRuntime::fixup_callers_callsite)));
 337   __ bl(rscratch1);
 338   __ maybe_isb();
 339 
 340   __ pop_CPU_state();
 341   // restore sp
 342   __ leave();
 343   __ bind(L);
 344 }
 345 
 346 static void gen_c2i_adapter(MacroAssembler *masm,
 347                             int total_args_passed,
 348                             int comp_args_on_stack,
 349                             const BasicType *sig_bt,
 350                             const VMRegPair *regs,
 351                             Label& skip_fixup) {
 352   // Before we get into the guts of the C2I adapter, see if we should be here
 353   // at all.  We've come from compiled code and are attempting to jump to the
 354   // interpreter, which means the caller made a static call to get here
 355   // (vcalls always get a compiled target if there is one).  Check for a
 356   // compiled target.  If there is one, we need to patch the caller's call.
 357   patch_callers_callsite(masm);
 358 
 359   __ bind(skip_fixup);
 360 
 361   // Since all args are passed on the stack, total_args_passed *
 362   // Interpreter::stackElementSize is the space we need.
 363 
 364   const int extraspace = total_args_passed * Interpreter::stackElementSize;
 365   const Register compArgPos = lr;
 366   int ld_shift = 0;
 367 
 368   __ str(compArgPos, Address(sp, -(extraspace + wordSize)));
 369   __ mov(compArgPos, sp);
 370 
 371   // Now write the args into the outgoing interpreter space
 372   for (int i = 0; i < total_args_passed; i++) {
 373 
 374     if (sig_bt[i] == T_VOID) {
 375       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 376       continue;
 377     }
 378 
 379     // next stack slot offset
 380     const int next_off = -Interpreter::stackElementSize;
 381 
 382     VMReg r_1 = regs[i].first();
 383     VMReg r_2 = regs[i].second();
 384     if (!r_1->is_valid()) {
 385       assert(!r_2->is_valid(), "");
 386       continue;
 387     }
 388 
 389     if (r_2->is_valid()) {
 390       assert(i + 1 < total_args_passed && sig_bt[i + 1] == T_VOID, "going to overrwrite reg_2 value");
 391     }
 392 
 393     if (r_1->is_stack()) {
 394       // memory to memory use rscratch1
 395       int ld_off = r_1->reg2stack() * VMRegImpl::stack_slot_size - ld_shift;
 396       if (!r_2->is_valid()) {
 397         __ ldr(rscratch1, Address(compArgPos, ld_off));
 398         __ str(rscratch1, Address(sp, next_off, Address::pre));
 399       } else {
 400           int tmp_off = ld_off;
 401           // ldrd accepts only imm8
 402           if(abs(ld_off) > (255 << 2)) {
 403               if(__ is_valid_for_imm12(ld_off)) {
 404                 __ add(compArgPos, compArgPos, ld_off);
 405               } else {
 406                 // add operates encoded imm12, NOT plain
 407                 __ mov(rscratch1, ld_off);
 408                 __ add(compArgPos, compArgPos, rscratch1);
 409               }
 410               tmp_off = 0;
 411               ld_shift += ld_off;
 412           }
 413         __ ldrd(rscratch1, rscratch2, Address(compArgPos, tmp_off));
 414         __ strd(rscratch1, rscratch2, Address(sp, 2* next_off, Address::pre));
 415       }
 416     } else if (r_1->is_Register()) {
 417       Register r = r_1->as_Register();
 418       assert(r != compArgPos, "compArgPos was modified");
 419       if (!r_2->is_valid()) {
 420         __ str(r, Address(sp, next_off, Address::pre));
 421       } else {
 422         assert(r_2->as_Register() != compArgPos, "compArgPos was modified");
 423         __ strd(r, r_2->as_Register(), Address(sp, 2 * next_off, Address::pre));
 424       }
 425     } else {
 426       assert(r_1->is_FloatRegister(), "");
 427       if (!r_2->is_valid()) {
 428         // Can't do pre or post addressing for vldr, vstr
 429         __ add(sp, sp, next_off);
 430         __ vstr_f32(r_1->as_FloatRegister(), Address(sp));
 431       } else {
 432     // TODO assert(r_2->is_FloatRegister() && r_2->as_FloatRegister() == r_1->as_FloatRegister() + 1, "");
 433         // Can't do pre or post addressing for vldr, vstr
 434         __ add(sp, sp, 2 * next_off);
 435         __ vstr_f64(r_1->as_FloatRegister(), Address(sp));
 436       }
 437     }
 438   }
 439 
 440   // hope, sp is returned to desired value
 441   __ ldr(compArgPos, Address(sp, -wordSize));
 442 
 443   // set sender sp
 444   if(__ is_valid_for_imm12(extraspace)) {
 445     __ add(r4, sp, extraspace);
 446   } else {
 447     __ mov(rscratch1, extraspace);
 448     __ add(r4, sp, rscratch1);
 449   }
 450 
 451   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::interpreter_entry_offset())));
 452   __ b(rscratch1);
 453 }
 454 
 455 static void range_check(MacroAssembler* masm, Register pc_reg, Register temp_reg,
 456                         address code_start, address code_end,
 457                         Label& L_ok) {
 458   Label L_fail;
 459   __ lea(temp_reg, ExternalAddress(code_start));
 460   __ cmp(pc_reg, temp_reg);
 461   __ b(L_fail, Assembler::LO);
 462   __ lea(temp_reg, ExternalAddress(code_end));
 463   __ cmp(pc_reg, temp_reg);
 464   __ b(L_ok, Assembler::LO);
 465   __ bind(L_fail);
 466 }
 467 
 468 static void gen_i2c_adapter(MacroAssembler *masm,
 469                             int total_args_passed,
 470                             int comp_args_on_stack,
 471                             const BasicType *sig_bt,
 472                             const VMRegPair *regs) {
 473 
 474   // Note: r13 contains the senderSP on entry. We must preserve it since
 475   // we may do a i2c -> c2i transition if we lose a race where compiled
 476   // code goes non-entrant while we get args ready.
 477 
 478   // In addition we use r13 to locate all the interpreter args because
 479   // we must align the stack to 16 bytes.
 480 
 481   // Adapters are frameless.
 482 
 483   // An i2c adapter is frameless because the *caller* frame, which is
 484   // interpreted, routinely repairs its own sp (from
 485   // interpreter_frame_last_sp), even if a callee has modified the
 486   // stack pointer.  It also recalculates and aligns sp.
 487 
 488   // A c2i adapter is frameless because the *callee* frame, which is
 489   // interpreted, routinely repairs its caller's sp (from sender_sp,
 490   // which is set up via the senderSP register).
 491 
 492   // In other words, if *either* the caller or callee is interpreted, we can
 493   // get the stack pointer repaired after a call.
 494 
 495   // This is why c2i and i2c adapters cannot be indefinitely composed.
 496   // In particular, if a c2i adapter were to somehow call an i2c adapter,
 497   // both caller and callee would be compiled methods, and neither would
 498   // clean up the stack pointer changes performed by the two adapters.
 499   // If this happens, control eventually transfers back to the compiled
 500   // caller, but with an uncorrected stack, causing delayed havoc.
 501 
 502   if (VerifyAdapterCalls &&
 503       (Interpreter::code() != NULL || StubRoutines::code1() != NULL)) {
 504     // So, let's test for cascading c2i/i2c adapters right now.
 505     //  assert(Interpreter::contains($return_addr) ||
 506     //         StubRoutines::contains($return_addr),
 507     //         "i2c adapter must return to an interpreter frame");
 508     __ block_comment("verify_i2c { ");
 509     Label L_ok;
 510     if (Interpreter::code() != NULL)
 511       range_check(masm, lr, rscratch1,
 512                   Interpreter::code()->code_start(), Interpreter::code()->code_end(),
 513                   L_ok);
 514     if (StubRoutines::code1() != NULL)
 515       range_check(masm, lr, rscratch1,
 516                   StubRoutines::code1()->code_begin(), StubRoutines::code1()->code_end(),
 517                   L_ok);
 518     if (StubRoutines::code2() != NULL)
 519       range_check(masm, lr, rscratch1,
 520                   StubRoutines::code2()->code_begin(), StubRoutines::code2()->code_end(),
 521                   L_ok);
 522     const char* msg = "i2c adapter must return to an interpreter frame";
 523     __ block_comment(msg);
 524     __ stop(msg);
 525     __ bind(L_ok);
 526     __ block_comment("} verify_i2ce ");
 527   }
 528 
 529   const int stack_space = round_to(comp_args_on_stack * VMRegImpl::stack_slot_size, StackAlignmentInBytes);
 530   const int ld_high = total_args_passed *Interpreter::stackElementSize;
 531   // Point to interpreter value (vs. tag)
 532   const int next_off =  -Interpreter::stackElementSize; // offset from ld ptr
 533   const Register loadCounter = lr;
 534 
 535   // Align sp to StackAlignmentInBytes so compiled frame starts always aligned
 536   // This is required by APCS, so all native code depends on it. The compiled
 537   // Java code is not required to follow this standard however doing so
 538   // simplifies the code because allows to have fixed size for compiled frames
 539   __ mov(rscratch2, sp);
 540   __ align_stack();
 541   if(total_args_passed) {
 542     // put below reserved stack space, imm12 should be enough
 543     __ str(loadCounter, Address(sp, -(stack_space + wordSize)));
 544 
 545     if(__ is_valid_for_imm12(ld_high)) {
 546         __ add(loadCounter, rscratch2, ld_high);
 547     } else {
 548         // add operates encoded imm12, we need plain
 549         __ mov(rscratch1, ld_high);
 550         __ add(loadCounter, rscratch2, rscratch1);
 551     }
 552   }
 553 
 554   if(comp_args_on_stack) {
 555     if(__ is_valid_for_imm12(stack_space)) {
 556         __ sub(sp, sp, stack_space);
 557     } else {
 558         // add operates encoded imm12, we need plain
 559         __ mov(rscratch1, stack_space);
 560         __ sub(sp, sp, rscratch1);
 561     }
 562   }
 563 
 564   // +------+   -> r4
 565   // |   0  | \
 566   // |   1  |  \
 567   // |   2  |   - >  Load in argument order going down.
 568   // |   x  |  /
 569   // |   N  | /
 570   // +------+ -> inital sp
 571   // | pad  | maybe 1 word to align the stack to 8 bytes
 572   // |   M  | \
 573   // |   x  |  \
 574   // |   2  |    ->  Load in argument order going up.
 575   // |   1  |  /
 576   // |   0  | /
 577   // +------+ ->
 578 
 579 
 580   int sp_offset = 0;
 581 
 582   // Now generate the shuffle code.
 583   for (int i = 0; i < total_args_passed; i++) {
 584 
 585     if (sig_bt[i] == T_VOID) {
 586       assert(i > 0 && (sig_bt[i-1] == T_LONG || sig_bt[i-1] == T_DOUBLE), "missing half");
 587       continue;
 588     }
 589 
 590     // Pick up 0, 1 or 2 words from SP+offset.
 591 
 592     //
 593     //
 594     //
 595     VMReg r_1 = regs[i].first();
 596     VMReg r_2 = regs[i].second();
 597     if (!r_1->is_valid()) {
 598       assert(!r_2->is_valid(), "");
 599       continue;
 600     }
 601 
 602     if (r_2->is_valid()) {
 603       assert(i + 1 < total_args_passed && sig_bt[i + 1] == T_VOID, "going to overrwrite reg_2 value");
 604     }
 605 
 606     if (r_1->is_stack()) {
 607       // Convert stack slot to an SP offset
 608       int st_off = regs[i].first()->reg2stack()*VMRegImpl::stack_slot_size - sp_offset;
 609 
 610       if (!r_2->is_valid()) {
 611         __ ldr(rscratch2, Address(loadCounter, next_off, Address::pre));
 612         __ str(rscratch2, Address(sp, st_off));
 613       } else {
 614         int tmp_off = st_off;
 615         if(abs(st_off) > (255 << 2)) {
 616             //st_off doesn't fit imm8 required by strd
 617 
 618             if(__ is_valid_for_imm12(st_off)) {
 619                 __ add(sp, sp, st_off);
 620             } else {
 621                 // add operates encoded imm12, NOT plain
 622                 __ mov(rscratch1, st_off);
 623                 __ add(sp, sp, rscratch1);
 624             }
 625             tmp_off = 0;
 626             sp_offset += st_off;
 627         }
 628 
 629 
 630         // Interpreter local[n] == MSW, local[n+1] == LSW however locals
 631         // are accessed as negative so LSW is at LOW address
 632 
 633         // this can be a misaligned move
 634     __ ldrd(rscratch1, rscratch2, Address(loadCounter, 2 * next_off, Address::pre));
 635     __ strd(rscratch1, rscratch2, Address(sp, tmp_off));
 636       }
 637     } else if (r_1->is_Register()) {  // Register argument
 638       Register r = r_1->as_Register();
 639       assert(r != loadCounter, "loadCounter is reloaded");
 640       if (r_2->is_valid()) {
 641         assert(r_2->as_Register() != loadCounter, "loadCounter is reloaded");
 642         // this can be a misaligned move
 643         // ldrd can handle inconsecutive registers
 644         __ ldrd(r, r_2->as_Register(), Address(loadCounter, 2 * next_off, Address::pre));
 645       } else {
 646         __ ldr(r, Address(loadCounter, next_off, Address::pre));
 647       }
 648     } else {
 649       assert(r_1->is_FloatRegister(), "");
 650       if (!r_2->is_valid()) {
 651         // Can't do pre or post addressing for vldr, vstr
 652         __ add(loadCounter, loadCounter, next_off);
 653         __ vldr_f32(r_1->as_FloatRegister(), Address(loadCounter));
 654       } else {
 655     // TODO assert(r_2->is_FloatRegister() && r_2->as_FloatRegister() == r_1->as_FloatRegister() + 1, "");
 656         // Can't do pre or post addressing for vldr, vstr
 657         __ add(loadCounter, loadCounter, 2 * next_off);
 658         __ vldr_f64(r_1->as_FloatRegister(), Address(loadCounter));
 659       }
 660     }
 661   }
 662 
 663   // restore sp
 664   if(sp_offset) {
 665     if(__ is_valid_for_imm12(sp_offset)) {
 666         __ sub(sp, sp, sp_offset);
 667     } else {
 668         // add operates encoded imm12, we need plain
 669         __ mov(rscratch1, sp_offset);
 670         __ sub(sp, sp, rscratch1);
 671     }
 672   }
 673 
 674   if(total_args_passed) {
 675     // restore loadCounter
 676     __ ldr(loadCounter, Address(sp, -wordSize));
 677   }
 678 
 679   // 6243940 We might end up in handle_wrong_method if
 680   // the callee is deoptimized as we race thru here. If that
 681   // happens we don't want to take a safepoint because the
 682   // caller frame will look interpreted and arguments are now
 683   // "compiled" so it is much better to make this transition
 684   // invisible to the stack walking code. Unfortunately if
 685   // we try and find the callee by normal means a safepoint
 686   // is possible. So we stash the desired callee in the thread
 687   // and the vm will find there should this case occur.
 688 
 689   __ str(rmethod, Address(rthread, JavaThread::callee_target_offset()));
 690 
 691   // Will jump to the compiled code just as if compiled code was doing it.
 692   __ ldr(rscratch1, Address(rmethod, in_bytes(Method::from_compiled_offset())));
 693   __ b(rscratch1);
 694 }
 695 
 696 // ---------------------------------------------------------------
 697 AdapterHandlerEntry* SharedRuntime::generate_i2c2i_adapters(MacroAssembler *masm,
 698                                                             int total_args_passed,
 699                                                             int comp_args_on_stack,
 700                                                             const BasicType *sig_bt,
 701                                                             const VMRegPair *regs,
 702                                                             AdapterFingerPrint* fingerprint) {
 703   address i2c_entry = __ pc();
 704   gen_i2c_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs);
 705 
 706   address c2i_unverified_entry = __ pc();
 707   Label skip_fixup;
 708 
 709   Label ok;
 710 
 711   Register holder = rscratch2;
 712   Register receiver = j_rarg0;
 713   Register tmp = r10;  // A call-clobbered register not used for arg passing
 714 
 715   // -------------------------------------------------------------------------
 716   // Generate a C2I adapter.  On entry we know rmethod holds the Method* during calls
 717   // to the interpreter.  The args start out packed in the compiled layout.  They
 718   // need to be unpacked into the interpreter layout.  This will almost always
 719   // require some stack space.  We grow the current (compiled) stack, then repack
 720   // the args.  We  finally end in a jump to the generic interpreter entry point.
 721   // On exit from the interpreter, the interpreter will restore our SP (lest the
 722   // compiled code, which relys solely on SP and not FP, get sick).
 723 
 724   {
 725     __ block_comment("c2i_unverified_entry {");
 726     __ load_klass(rscratch1, receiver);
 727     __ ldr(tmp, Address(holder, CompiledICHolder::holder_klass_offset()));
 728     __ cmp(rscratch1, tmp);
 729     __ ldr(rmethod, Address(holder, CompiledICHolder::holder_metadata_offset()));
 730     __ b(ok, Assembler::EQ);
 731     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 732 
 733     __ bind(ok);
 734     // Method might have been compiled since the call site was patched to
 735     // interpreted; if that is the case treat it as a miss so we can get
 736     // the call site corrected.
 737     __ ldr(rscratch1, Address(rmethod, in_bytes(Method::code_offset())));
 738     __ cbz(rscratch1, skip_fixup);
 739     __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
 740     __ block_comment("} c2i_unverified_entry");
 741   }
 742 
 743   address c2i_entry = __ pc();
 744 
 745   gen_c2i_adapter(masm, total_args_passed, comp_args_on_stack, sig_bt, regs, skip_fixup);
 746 
 747   __ flush();
 748   return AdapterHandlerLibrary::new_entry(fingerprint, i2c_entry, c2i_entry, c2i_unverified_entry);
 749 }
 750 
 751 int SharedRuntime::c_calling_convention(const BasicType *sig_bt,
 752                                          VMRegPair *regs,
 753                                          VMRegPair *regs2,
 754                                          int total_args_passed) {
 755   assert(regs2 == NULL, "not needed on AArch32");
 756 
 757 // We return the amount of VMRegImpl stack slots we need to reserve for all
 758 // the arguments NOT counting out_preserve_stack_slots.
 759 
 760     static const Register INT_ArgReg[Argument::n_int_register_parameters_c] = {
 761       c_rarg0, c_rarg1, c_rarg2, c_rarg3
 762     };
 763 #ifdef HARD_FLOAT_CC
 764     const int FP_ArgReg_N = 16;
 765     static const FloatRegister FP_ArgReg[] = {
 766       f0, f1, f2, f3,
 767       f4, f5, f6, f7,
 768       f8, f9, f10, f11,
 769       f12, f13, f14, f15,
 770     };
 771     unsigned long fp_free_mask = (1 << FP_ArgReg_N) - 1;
 772     uint fp_args = 0;
 773 #endif //HARD_FLOAT_CC
 774 
 775     uint int_args = 0;
 776     uint stk_args = 0;
 777 
 778     for (int i = 0; i < total_args_passed; i++) {
 779       switch (sig_bt[i]) {
 780       case T_BOOLEAN:
 781       case T_CHAR:
 782       case T_BYTE:
 783       case T_SHORT:
 784       case T_INT:
 785       case T_OBJECT:
 786       case T_ARRAY:
 787       case T_ADDRESS:
 788       case T_METADATA:
 789 #ifndef HARD_FLOAT_CC
 790       // soft FP case
 791       case T_FLOAT:
 792 #endif
 793         if (int_args < Argument::n_int_register_parameters_c) {
 794           regs[i].set1(INT_ArgReg[int_args++]->as_VMReg());
 795         } else {
 796           regs[i].set1(VMRegImpl::stack2reg(stk_args));
 797           stk_args += 1;
 798         }
 799         break;
 800 #ifndef HARD_FLOAT_CC
 801       // soft FP case
 802       case  T_DOUBLE:
 803 #endif
 804       case T_LONG:
 805         assert(sig_bt[i + 1] == T_VOID, "expecting half");
 806         if (int_args + 1 < Argument::n_int_register_parameters_c) {
 807             // c2 requires aligned reg pair
 808             int_args = round_to(int_args, 2);
 809             regs[i].set2(INT_ArgReg[int_args]->as_VMReg());
 810             int_args += 2;
 811         } else {
 812             stk_args = round_to(stk_args, 2);
 813             regs[i].set2(VMRegImpl::stack2reg(stk_args));
 814             stk_args += 2;
 815             // close "arg at reg" branch
 816             int_args = Argument::n_int_register_parameters_c;
 817         }
 818         break;
 819 #ifdef HARD_FLOAT_CC
 820       case T_FLOAT:
 821         if (fp_free_mask & ((1 << FP_ArgReg_N)-1)) {
 822           unsigned index = __builtin_ctz(fp_free_mask);
 823           regs[i].set1(FP_ArgReg[index]->as_VMReg());
 824           fp_free_mask &= ~(1 << index);
 825           fp_args += 2 * ((~index) & 1);
 826         } else {
 827           regs[i].set1(VMRegImpl::stack2reg(stk_args));
 828           stk_args += 1;
 829         }
 830         break;
 831       case T_DOUBLE:
 832         assert(sig_bt[i + 1] == T_VOID, "expecting half");
 833         assert(!(fp_args & 1), "must be aligned");
 834         if (fp_args + 1 < FP_ArgReg_N) {
 835           fp_free_mask &= ~(3 << fp_args);
 836           regs[i].set2(FP_ArgReg[fp_args]->as_VMReg());
 837           fp_args += 2;
 838         } else {
 839           fp_free_mask = 0; // make future float allocations on stack too
 840           stk_args = round_to(stk_args, 2);
 841           regs[i].set2(VMRegImpl::stack2reg(stk_args));
 842           stk_args += 2;
 843         }
 844         break;
 845 #endif //HARD_FLOAT_CC
 846       case T_VOID: // Halves of longs and doubles
 847         assert(i != 0 && (sig_bt[i - 1] == T_LONG || sig_bt[i - 1] == T_DOUBLE), "expecting half");
 848         regs[i].set_bad();
 849         break;
 850       default:
 851         ShouldNotReachHere();
 852         break;
 853       }
 854     }
 855 
 856   return round_to(stk_args, StackAlignmentInBytes/wordSize);
 857 }
 858 
 859 // On 64 bit we will store integer like items to the stack as
 860 // 64 bits items (sparc abi) even though java would only store
 861 // 32bits for a parameter. On 32bit it will simply be 32 bits
 862 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
 863 
 864 static void move_int(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
 865   if (src.first()->is_stack()) {
 866     if (dst.first()->is_stack()) {
 867       // stack to stack
 868       __ ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
 869       __ str(rscratch1, Address(sp, reg2offset_out(dst.first())));
 870     } else {
 871       // stack to reg
 872       __ ldr(dst.first()->as_Register(), Address(rfp, reg2offset_in(src.first())));
 873     }
 874   } else if (dst.first()->is_stack()) {
 875     // reg to stack
 876     __ str(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first())));
 877   } else {
 878     if (dst.first() != src.first()) {
 879       __ mov(dst.first()->as_Register(), src.first()->as_Register());
 880     }
 881   }
 882 }
 883 
 884 // An oop arg. Must pass a handle not the oop itself
 885 static void object_move(MacroAssembler* masm,
 886                         OopMap* map,
 887                         int oop_handle_offset,
 888                         int framesize_in_slots,
 889                         VMRegPair src,
 890                         VMRegPair dst,
 891                         bool is_receiver,
 892                         int* receiver_offset) {
 893 
 894   // must pass a handle. First figure out the location we use as a handle
 895 
 896   Register rHandle = dst.first()->is_stack() ? rscratch2 : dst.first()->as_Register();
 897 
 898   // See if oop is NULL if it is we need no handle
 899 
 900   if (src.first()->is_stack()) {
 901 
 902     // Oop is already on the stack as an argument
 903     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
 904     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
 905     if (is_receiver) {
 906       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
 907     }
 908 
 909     __ ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
 910     __ lea(rHandle, Address(rfp, reg2offset_in(src.first())));
 911     // conditionally move a NULL
 912     __ cmp(rscratch1, 0);
 913     __ mov(rHandle, 0, Assembler::EQ);
 914   } else {
 915 
 916     // Oop is in an a register we must store it to the space we reserve
 917     // on the stack for oop_handles and pass a handle if oop is non-NULL
 918 
 919     const Register rOop = src.first()->as_Register();
 920     int oop_slot;
 921     if (rOop == j_rarg0)
 922       oop_slot = 0;
 923     else if (rOop == j_rarg1)
 924       oop_slot = 1;
 925     else if (rOop == j_rarg2)
 926       oop_slot = 2;
 927     else {
 928       assert(rOop == j_rarg3, "wrong register");
 929       oop_slot = 3;
 930     }
 931 
 932     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
 933     int offset = oop_slot*VMRegImpl::stack_slot_size;
 934 
 935     map->set_oop(VMRegImpl::stack2reg(oop_slot));
 936     // Store oop in handle area, may be NULL
 937     __ str(rOop, Address(sp, offset));
 938     if (is_receiver) {
 939       *receiver_offset = offset;
 940     }
 941 
 942     __ cmp(rOop, 0);
 943     __ lea(rHandle, Address(sp, offset));
 944     // conditionally move a NULL
 945     __ mov(rHandle, 0, Assembler::EQ);
 946   }
 947 
 948   // If arg is on the stack then place it otherwise it is already in correct reg.
 949   if (dst.first()->is_stack()) {
 950     __ str(rHandle, Address(sp, reg2offset_out(dst.first())));
 951   }
 952 }
 953 
 954 // A float arg may have to do float reg int reg conversion
 955 static void float_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
 956     if(hasFPU()) {
 957         if (src.first()->is_stack()) {
 958           if (dst.first()->is_stack()) {
 959             // stack to stack
 960             // Have no vfp scratch registers, so copy via gpr
 961             __ ldr(rscratch1, Address(rfp, reg2offset_in(src.first())));
 962             __ str(rscratch1, Address(sp, reg2offset_out(dst.first())));
 963           } else {
 964             // stack to reg
 965             __ vldr_f32(dst.first()->as_FloatRegister(), Address(rfp, reg2offset_in(src.first())));
 966           }
 967         } else if (dst.first()->is_stack()) {
 968           // reg to stack
 969           __ vstr_f32(src.first()->as_FloatRegister(), Address(sp, reg2offset_out(dst.first())));
 970         } else {
 971 #ifndef HARD_FLOAT_CC
 972             if(dst.first()->is_Register()) {
 973                 __ vmov_f32(dst.first()->as_Register(), src.first()->as_FloatRegister());
 974             } else
 975 #endif
 976             if (dst.first() != src.first()) {
 977                  __ vmov_f32(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
 978             }
 979         }
 980     } else {
 981         move_int(masm, src, dst);
 982     }
 983 }
 984 
 985 // A long move
 986 static void long_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
 987   if (src.first()->is_stack()) {
 988     if (dst.first()->is_stack()) {
 989       // stack to stack
 990       __ ldrd(rscratch1, rscratch2, Address(rfp, reg2offset_in(src.first())));
 991       __ strd(rscratch1, rscratch2, Address(sp, reg2offset_out(dst.first())));
 992     } else {
 993       // stack to reg
 994       __ ldrd(dst.first()->as_Register(), dst.second()->as_Register(),
 995       Address(rfp, reg2offset_in(src.first())));
 996     }
 997   } else if (dst.first()->is_stack()) {
 998     // reg to stack
 999     __ strd(src.first()->as_Register(), src.second()->as_Register(),
1000     Address(sp, reg2offset_out(dst.first())));
1001   } else {
1002     // reg to reg
1003     if (dst.first() != src.first()) {
1004       if (dst.first() != src.second()) {
1005         __ mov(dst.first()->as_Register(), src.first()->as_Register());
1006         __ mov(dst.second()->as_Register(), src.second()->as_Register());
1007       } else {
1008         __ mov(dst.second()->as_Register(), src.second()->as_Register());
1009         __ mov(dst.first()->as_Register(), src.first()->as_Register());
1010       }
1011     }
1012   }
1013 }
1014 
1015 // A double move
1016 static void double_move(MacroAssembler* masm, VMRegPair src, VMRegPair dst) {
1017   if(hasFPU()) {
1018     if (src.first()->is_stack()) {
1019       if (dst.first()->is_stack()) {
1020         // stack to stack
1021         // Have no vfp scratch registers, so copy via gpr
1022         __ ldrd(rscratch1, rscratch2, Address(rfp, reg2offset_in(src.first())));
1023         __ strd(rscratch1, rscratch2, Address(sp, reg2offset_out(dst.first())));
1024       } else {
1025         // stack to reg
1026         __ vldr_f64(dst.first()->as_FloatRegister(), Address(rfp, reg2offset_in(src.first())));
1027       }
1028     } else if (dst.first()->is_stack()) {
1029       // reg to stack
1030       __ vstr_f64(src.first()->as_FloatRegister(), Address(sp, reg2offset_out(dst.first())));
1031     } else {
1032 #ifndef HARD_FLOAT_CC
1033         if(dst.first()->is_Register()) {
1034             __ vmov_f64(dst.first()->as_Register(), dst.second()->as_Register(), src.first()->as_FloatRegister());
1035         } else
1036 #endif
1037         if (dst.first() != src.first()) {
1038            __ vmov_f64(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
1039         }
1040       }
1041   } else {
1042     long_move(masm, src, dst);
1043   }
1044 }
1045 
1046 
1047 void SharedRuntime::save_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1048   // We always ignore the frame_slots arg and just use the space just below frame pointer
1049   // which by this time is free to use
1050   switch (ret_type) {
1051   case T_DOUBLE:
1052 #ifdef HARD_FLOAT_CC
1053     __ vstr_f64(d0, Address(rfp, -3 * wordSize));
1054     break;
1055 #endif//fall through otherwise
1056   case T_LONG:
1057     __ strd(r0, r1, Address(rfp, -3 * wordSize));
1058     break;
1059   case T_VOID:
1060     break;
1061   case T_FLOAT:
1062 #ifdef HARD_FLOAT_CC
1063     __ vstr_f32(f0, Address(rfp, -2 * wordSize));
1064     break;
1065 #endif//fall through otherwise
1066   default:
1067     __ str(r0, Address(rfp, -2 * wordSize));
1068     break;
1069   }
1070 }
1071 
1072 void SharedRuntime::restore_native_result(MacroAssembler *masm, BasicType ret_type, int frame_slots) {
1073   // We always ignore the frame_slots arg and just use the space just below frame pointer
1074   // which by this time is free to use
1075   switch (ret_type) {
1076   case T_DOUBLE:
1077 #ifdef HARD_FLOAT_CC
1078     __ vldr_f64(d0, Address(rfp, -3 * wordSize));
1079     break;
1080 #endif//fall through otherwise
1081   case T_LONG:
1082     __ ldrd(r0, r1, Address(rfp, -3 * wordSize));
1083     break;
1084   case T_VOID:
1085     break;
1086   case T_FLOAT:
1087 #ifdef HARD_FLOAT_CC
1088     __ vldr_f32(d0, Address(rfp, -2 * wordSize));
1089     break;
1090 #endif//fall through otherwise
1091   default:
1092     __ ldr(r0, Address(rfp, -2 * wordSize));
1093     break;
1094   }
1095 }
1096 
1097 static int save_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
1098   RegSet x;
1099   int saved_slots = 0;
1100   for ( int i = first_arg ; i < arg_count ; i++ ) {
1101     if (args[i].first()->is_Register()) {
1102       x = x + args[i].first()->as_Register();
1103       ++saved_slots;
1104       if (args[i].second()->is_valid()) {
1105           x = x + args[i].second()->as_Register();
1106           ++saved_slots;
1107       }
1108     }
1109 #ifdef HARD_FLOAT_CC
1110     else if (args[i].first()->is_FloatRegister()) {
1111       FloatRegister fr = args[i].first()->as_FloatRegister();
1112 
1113       if (args[i].second()->is_FloatRegister()) {
1114     assert(args[i].is_single_phys_reg(), "doubles should be 2 consequents float regs");
1115         __ decrement(sp, 2 * wordSize);
1116     __ vstr_f64(fr, Address(sp));
1117         saved_slots += 2;
1118       } else {
1119         __ decrement(sp, wordSize);
1120     __ vstr_f32(fr, Address(sp));
1121         ++saved_slots;
1122       }
1123     }
1124 #endif//HARD_FLOAT_CC
1125   }
1126   __ push(x, sp);
1127   return saved_slots;
1128 }
1129 
1130 static void restore_args(MacroAssembler *masm, int arg_count, int first_arg, VMRegPair *args) {
1131   RegSet x;
1132   for ( int i = first_arg ; i < arg_count ; i++ ) {
1133     if (args[i].first()->is_Register()) {
1134       x = x + args[i].first()->as_Register();
1135       if (args[i].second()->is_valid()) {
1136         x = x + args[i].second()->as_Register();
1137       }
1138     } else {
1139       ;
1140     }
1141   }
1142   __ pop(x, sp);
1143   for ( int i = arg_count - 1 ; i >= first_arg ; i-- ) {
1144     if (args[i].first()->is_Register()) {
1145       ;
1146     }
1147 #ifdef HARD_FLOAT_CC
1148     else if (args[i].first()->is_FloatRegister()) {
1149       FloatRegister fr = args[i].first()->as_FloatRegister();
1150 
1151       if (args[i].second()->is_FloatRegister()) {
1152     assert(args[i].is_single_phys_reg(), "doubles should be 2 consequents float regs");
1153     __ vldr_f64(fr, Address(sp));
1154         __ increment(sp, 2 * wordSize);
1155       } else {
1156     __ vldr_f32(fr, Address(sp));
1157         __ increment(sp, wordSize);
1158       }
1159     }
1160 #endif//HARD_FLOAT_CC
1161   }
1162 }
1163 
1164 
1165 // Check GC_locker::needs_gc and enter the runtime if it's true.  This
1166 // keeps a new JNI critical region from starting until a GC has been
1167 // forced.  Save down any oops in registers and describe them in an
1168 // OopMap.
1169 static void check_needs_gc_for_critical_native(MacroAssembler* masm,
1170                                                int stack_slots,
1171                                                int total_c_args,
1172                                                int total_in_args,
1173                                                int arg_save_area,
1174                                                OopMapSet* oop_maps,
1175                                                VMRegPair* in_regs,
1176                                                BasicType* in_sig_bt) {
1177   __ block_comment("check GC_locker::needs_gc");
1178   Label cont;
1179   __ lea(rscratch1, ExternalAddress((address)GC_locker::needs_gc_address()));
1180   __ ldr(rscratch1, Address(rscratch1));
1181   __ cmp(rscratch1, false);
1182   __ b(cont, Assembler::EQ);
1183 
1184   // Save down any incoming oops and call into the runtime to halt for a GC
1185 
1186   OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1187 
1188   save_args(masm, total_in_args, 0, in_regs);
1189 
1190   address the_pc = __ pc();
1191   oop_maps->add_gc_map( __ offset(), map);
1192   __ set_last_Java_frame(sp, noreg, the_pc, rscratch1);
1193 
1194   __ block_comment("block_for_jni_critical");
1195   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::block_for_jni_critical), rthread);
1196 
1197   __ reset_last_Java_frame(true);
1198 
1199   restore_args(masm, total_in_args, 0, in_regs);
1200 
1201   __ bind(cont);
1202 #ifdef ASSERT
1203   if (StressCriticalJNINatives) {
1204     // Stress register saving
1205     OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1206     save_args(masm, total_in_args, 0, in_regs);
1207 
1208     // Destroy argument registers.
1209     for (int i = 0; i < total_in_args; i++) {
1210       if (in_regs[i].first()->is_Register()) {
1211         Register reg = in_regs[i].first()->as_Register();
1212         __ neg(reg, reg);
1213         if (in_regs[i].second()->is_Register()) {
1214           reg = in_regs[i].second()->as_Register();
1215           __ neg(reg, reg);
1216         }
1217       } else if (in_regs[i].first()->is_FloatRegister()) {
1218         FloatRegister freg = in_regs[i].first()->as_FloatRegister();
1219         __ vneg_f32(freg, freg);
1220         if (in_regs[i].second()->is_FloatRegister()) {
1221           FloatRegister freg = in_regs[i].second()->as_FloatRegister();
1222           __ vneg_f32(freg, freg);
1223         }
1224       }
1225     }
1226 
1227     restore_args(masm, total_in_args, 0, in_regs);
1228   }
1229 #endif
1230 }
1231 
1232 // Unpack an array argument into a pointer to the body and the length
1233 // if the array is non-null, otherwise pass 0 for both.
1234 static void unpack_array_argument(MacroAssembler* masm, VMRegPair reg, BasicType in_elem_type, VMRegPair body_arg, VMRegPair length_arg, Register tmp_reg) {
1235   assert(!body_arg.first()->is_Register() || body_arg.first()->as_Register() != tmp_reg,
1236          "possible collision");
1237   assert(!length_arg.first()->is_Register() || length_arg.first()->as_Register() != tmp_reg,
1238          "possible collision");
1239 
1240   __ block_comment("unpack_array_argument {");
1241 
1242   // Pass the length, ptr pair
1243   Label is_null, done;
1244   VMRegPair tmp;
1245   tmp.set_ptr(tmp_reg->as_VMReg());
1246   if (reg.first()->is_stack()) {
1247     // Load the arg up from the stack
1248     move_int(masm, reg, tmp);
1249     reg = tmp;
1250   }
1251   __ cbz(reg.first()->as_Register(), is_null);
1252   __ lea(tmp_reg, Address(reg.first()->as_Register(), arrayOopDesc::base_offset_in_bytes(in_elem_type)));
1253   move_int(masm, tmp, body_arg);
1254   // load the length relative to the body.
1255   __ ldr(tmp_reg, Address(tmp_reg, arrayOopDesc::length_offset_in_bytes() -
1256                            arrayOopDesc::base_offset_in_bytes(in_elem_type)));
1257   move_int(masm, tmp, length_arg);
1258   __ b(done);
1259   __ bind(is_null);
1260   // Pass zeros
1261   __ movptr(tmp_reg, 0);
1262   move_int(masm, tmp, body_arg);
1263   move_int(masm, tmp, length_arg);
1264   __ bind(done);
1265 
1266   __ block_comment("} unpack_array_argument");
1267 
1268 }
1269 
1270 // Different signatures may require very different orders for the move
1271 // to avoid clobbering other arguments.  There's no simple way to
1272 // order them safely.  Compute a safe order for issuing stores and
1273 // break any cycles in those stores.  This code is fairly general but
1274 // it's not necessary on the other platforms so we keep it in the
1275 // platform dependent code instead of moving it into a shared file.
1276 // (See bugs 7013347 & 7145024.)
1277 class ComputeMoveOrder: public StackObj {
1278   class MoveOperation: public ResourceObj {
1279     friend class ComputeMoveOrder;
1280    private:
1281     VMRegPair        _src;
1282     VMRegPair        _dst;
1283     int              _src_index;
1284     int              _dst_index;
1285     bool             _processed;
1286     MoveOperation*  _next;
1287     MoveOperation*  _prev;
1288 
1289     static int get_id(VMRegPair r) {
1290       return r.first()->value();
1291     }
1292 
1293    public:
1294     MoveOperation(int src_index, VMRegPair src, int dst_index, VMRegPair dst):
1295       _src(src)
1296     , _src_index(src_index)
1297     , _dst(dst)
1298     , _dst_index(dst_index)
1299     , _next(NULL)
1300     , _prev(NULL)
1301     , _processed(false) {
1302     }
1303 
1304     VMRegPair src() const              { return _src; }
1305     int src_id() const                 { return get_id(src()); }
1306     int src_index() const              { return _src_index; }
1307     VMRegPair dst() const              { return _dst; }
1308     void set_dst(int i, VMRegPair dst) { _dst_index = i, _dst = dst; }
1309     int dst_index() const              { return _dst_index; }
1310     int dst_id() const                 { return get_id(dst()); }
1311     MoveOperation* next() const       { return _next; }
1312     MoveOperation* prev() const       { return _prev; }
1313     void set_processed()               { _processed = true; }
1314     bool is_processed() const          { return _processed; }
1315 
1316     // insert
1317     void break_cycle(VMRegPair temp_register) {
1318       // create a new store following the last store
1319       // to move from the temp_register to the original
1320       MoveOperation* new_store = new MoveOperation(-1, temp_register, dst_index(), dst());
1321 
1322       // break the cycle of links and insert new_store at the end
1323       // break the reverse link.
1324       MoveOperation* p = prev();
1325       assert(p->next() == this, "must be");
1326       _prev = NULL;
1327       p->_next = new_store;
1328       new_store->_prev = p;
1329 
1330       // change the original store to save it's value in the temp.
1331       set_dst(-1, temp_register);
1332     }
1333 
1334     void link(GrowableArray<MoveOperation*>& killer) {
1335       // link this store in front the store that it depends on
1336       MoveOperation* n = killer.at_grow(src_id(), NULL);
1337       if (n != NULL) {
1338         assert(_next == NULL && n->_prev == NULL, "shouldn't have been set yet");
1339         _next = n;
1340         n->_prev = this;
1341       }
1342     }
1343   };
1344 
1345  private:
1346   GrowableArray<MoveOperation*> edges;
1347 
1348  public:
1349   ComputeMoveOrder(int total_in_args, VMRegPair* in_regs, int total_c_args, VMRegPair* out_regs,
1350                     BasicType* in_sig_bt, GrowableArray<int>& arg_order, VMRegPair tmp_vmreg) {
1351     // Move operations where the dest is the stack can all be
1352     // scheduled first since they can't interfere with the other moves.
1353     for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
1354       if (in_sig_bt[i] == T_ARRAY) {
1355         c_arg--;
1356         if (out_regs[c_arg].first()->is_stack() &&
1357             out_regs[c_arg + 1].first()->is_stack()) {
1358           arg_order.push(i);
1359           arg_order.push(c_arg);
1360         } else {
1361           if (out_regs[c_arg].first()->is_stack() ||
1362               in_regs[i].first() == out_regs[c_arg].first()) {
1363             add_edge(i, in_regs[i].first(), c_arg, out_regs[c_arg + 1]);
1364           } else {
1365             add_edge(i, in_regs[i].first(), c_arg, out_regs[c_arg]);
1366           }
1367         }
1368       } else if (in_sig_bt[i] == T_VOID) {
1369         arg_order.push(i);
1370         arg_order.push(c_arg);
1371       } else {
1372         if (out_regs[c_arg].first()->is_stack() ||
1373             in_regs[i].first() == out_regs[c_arg].first()) {
1374           arg_order.push(i);
1375           arg_order.push(c_arg);
1376         } else {
1377           add_edge(i, in_regs[i].first(), c_arg, out_regs[c_arg]);
1378         }
1379       }
1380     }
1381     // Break any cycles in the register moves and emit the in the
1382     // proper order.
1383     GrowableArray<MoveOperation*>* stores = get_store_order(tmp_vmreg);
1384     for (int i = 0; i < stores->length(); i++) {
1385       arg_order.push(stores->at(i)->src_index());
1386       arg_order.push(stores->at(i)->dst_index());
1387     }
1388  }
1389 
1390   // Collected all the move operations
1391   void add_edge(int src_index, VMRegPair src, int dst_index, VMRegPair dst) {
1392     if (src.first() == dst.first()) return;
1393     edges.append(new MoveOperation(src_index, src, dst_index, dst));
1394   }
1395 
1396   // Walk the edges breaking cycles between moves.  The result list
1397   // can be walked in order to produce the proper set of loads
1398   GrowableArray<MoveOperation*>* get_store_order(VMRegPair temp_register) {
1399     // Record which moves kill which values
1400     GrowableArray<MoveOperation*> killer;
1401     for (int i = 0; i < edges.length(); i++) {
1402       MoveOperation* s = edges.at(i);
1403       assert(killer.at_grow(s->dst_id(), NULL) == NULL, "only one killer");
1404       killer.at_put_grow(s->dst_id(), s, NULL);
1405     }
1406     assert(killer.at_grow(MoveOperation::get_id(temp_register), NULL) == NULL,
1407            "make sure temp isn't in the registers that are killed");
1408 
1409     // create links between loads and stores
1410     for (int i = 0; i < edges.length(); i++) {
1411       edges.at(i)->link(killer);
1412     }
1413 
1414     // at this point, all the move operations are chained together
1415     // in a doubly linked list.  Processing it backwards finds
1416     // the beginning of the chain, forwards finds the end.  If there's
1417     // a cycle it can be broken at any point,  so pick an edge and walk
1418     // backward until the list ends or we end where we started.
1419     GrowableArray<MoveOperation*>* stores = new GrowableArray<MoveOperation*>();
1420     for (int e = 0; e < edges.length(); e++) {
1421       MoveOperation* s = edges.at(e);
1422       if (!s->is_processed()) {
1423         MoveOperation* start = s;
1424         // search for the beginning of the chain or cycle
1425         while (start->prev() != NULL && start->prev() != s) {
1426           start = start->prev();
1427         }
1428         if (start->prev() == s) {
1429           start->break_cycle(temp_register);
1430         }
1431         // walk the chain forward inserting to store list
1432         while (start != NULL) {
1433           stores->append(start);
1434           start->set_processed();
1435           start = start->next();
1436         }
1437       }
1438     }
1439     return stores;
1440   }
1441 };
1442 
1443 
1444 static void rt_call(MacroAssembler* masm, address dest) {
1445   CodeBlob *cb = CodeCache::find_blob(dest);
1446   if (cb) {
1447     __ far_call(RuntimeAddress(dest), NULL, rscratch2);
1448   } else {
1449     __ lea(rscratch2, RuntimeAddress(dest));
1450     __ bl(rscratch2);
1451     __ maybe_isb();
1452   }
1453 }
1454 
1455 static void verify_oop_args(MacroAssembler* masm,
1456                             methodHandle method,
1457                             const BasicType* sig_bt,
1458                             const VMRegPair* regs) {
1459   Register temp_reg = rscratch2;  // not part of any compiled calling seq
1460   if (VerifyOops) {
1461     for (int i = 0; i < method->size_of_parameters(); i++) {
1462       if (sig_bt[i] == T_OBJECT ||
1463           sig_bt[i] == T_ARRAY) {
1464         VMReg r = regs[i].first();
1465         assert(r->is_valid(), "bad oop arg");
1466         if (r->is_stack()) {
1467           __ ldr(temp_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size));
1468           __ verify_oop(temp_reg);
1469         } else {
1470           __ verify_oop(r->as_Register());
1471         }
1472       }
1473     }
1474   }
1475 }
1476 
1477 static void gen_special_dispatch(MacroAssembler* masm,
1478                                  methodHandle method,
1479                                  const BasicType* sig_bt,
1480                                  const VMRegPair* regs) {
1481   verify_oop_args(masm, method, sig_bt, regs);
1482   vmIntrinsics::ID iid = method->intrinsic_id();
1483 
1484   // Now write the args into the outgoing interpreter space
1485   bool     has_receiver   = false;
1486   Register receiver_reg   = noreg;
1487   int      member_arg_pos = -1;
1488   Register member_reg     = noreg;
1489   int      ref_kind       = MethodHandles::signature_polymorphic_intrinsic_ref_kind(iid);
1490   if (ref_kind != 0) {
1491     member_arg_pos = method->size_of_parameters() - 1;  // trailing MemberName argument
1492     member_reg = r4;
1493     has_receiver = MethodHandles::ref_kind_has_receiver(ref_kind);
1494   } else if (iid == vmIntrinsics::_invokeBasic) {
1495     has_receiver = true;
1496   } else {
1497     fatal(err_msg_res("unexpected intrinsic id %d", iid));
1498   }
1499 
1500   if (member_reg != noreg) {
1501     // Load the member_arg into register, if necessary.
1502     SharedRuntime::check_member_name_argument_is_last_argument(method, sig_bt, regs);
1503     VMReg r = regs[member_arg_pos].first();
1504     if (r->is_stack()) {
1505       __ ldr(member_reg, Address(sp, r->reg2stack() * VMRegImpl::stack_slot_size));
1506     } else {
1507       // no data motion is needed
1508       member_reg = r->as_Register();
1509     }
1510   }
1511 
1512   if (has_receiver) {
1513     // Make sure the receiver is loaded into a register.
1514     assert(method->size_of_parameters() > 0, "oob");
1515     assert(sig_bt[0] == T_OBJECT, "receiver argument must be an object");
1516     VMReg r = regs[0].first();
1517     assert(r->is_valid(), "bad receiver arg");
1518     if (r->is_stack()) {
1519       // Porting note:  This assumes that compiled calling conventions always
1520       // pass the receiver oop in a register.  If this is not true on some
1521       // platform, pick a temp and load the receiver from stack.
1522       fatal("receiver always in a register");
1523     } else {
1524       // no data motion is needed
1525       receiver_reg = r->as_Register();
1526     }
1527   }
1528 
1529   // Figure out which address we are really jumping to:
1530   MethodHandles::generate_method_handle_dispatch(masm, iid,
1531                                                  receiver_reg, member_reg, /*for_compiler_entry:*/ true);
1532 }
1533 
1534 // ---------------------------------------------------------------------------
1535 // Generate a native wrapper for a given method.  The method takes arguments
1536 // in the Java compiled code convention, marshals them to the native
1537 // convention (handlizes oops, etc), transitions to native, makes the call,
1538 // returns to java state (possibly blocking), unhandlizes any result and
1539 // returns.
1540 //
1541 // Critical native functions are a shorthand for the use of
1542 // GetPrimtiveArrayCritical and disallow the use of any other JNI
1543 // functions.  The wrapper is expected to unpack the arguments before
1544 // passing them to the callee and perform checks before and after the
1545 // native call to ensure that they GC_locker
1546 // lock_critical/unlock_critical semantics are followed.  Some other
1547 // parts of JNI setup are skipped like the tear down of the JNI handle
1548 // block and the check for pending exceptions it's impossible for them
1549 // to be thrown.
1550 //
1551 // They are roughly structured like this:
1552 //    if (GC_locker::needs_gc())
1553 //      SharedRuntime::block_for_jni_critical();
1554 //    tranistion to thread_in_native
1555 //    unpack arrray arguments and call native entry point
1556 //    check for safepoint in progress
1557 //    check if any thread suspend flags are set
1558 //      call into JVM and possible unlock the JNI critical
1559 //      if a GC was suppressed while in the critical native.
1560 //    transition back to thread_in_Java
1561 //    return to caller
1562 //
1563 nmethod* SharedRuntime::generate_native_wrapper(MacroAssembler* masm,
1564                                                 methodHandle method,
1565                                                 int compile_id,
1566                                                 BasicType* in_sig_bt,
1567                                                 VMRegPair* in_regs,
1568                                                 BasicType ret_type) {
1569   if (method->is_method_handle_intrinsic()) {
1570     vmIntrinsics::ID iid = method->intrinsic_id();
1571     intptr_t start = (intptr_t)__ pc();
1572     int vep_offset = ((intptr_t)__ pc()) - start;
1573 
1574     // First instruction must be a nop as it may need to be patched on deoptimisation
1575     __ nop();
1576     gen_special_dispatch(masm,
1577                          method,
1578                          in_sig_bt,
1579                          in_regs);
1580     int frame_complete = ((intptr_t)__ pc()) - start;  // not complete, period
1581     __ flush();
1582     int stack_slots = SharedRuntime::out_preserve_stack_slots();  // no out slots at all, actually
1583     return nmethod::new_native_nmethod(method,
1584                                        compile_id,
1585                                        masm->code(),
1586                                        vep_offset,
1587                                        frame_complete,
1588                                        stack_slots / VMRegImpl::slots_per_word,
1589                                        in_ByteSize(-1),
1590                                        in_ByteSize(-1),
1591                                        (OopMapSet*)NULL);
1592   }
1593 
1594   bool is_critical_native = true;
1595   address native_func = method->critical_native_function();
1596   if (native_func == NULL) {
1597     native_func = method->native_function();
1598     is_critical_native = false;
1599   }
1600   assert(native_func != NULL, "must have function");
1601 
1602   // An OopMap for lock (and class if static)
1603   OopMapSet *oop_maps = new OopMapSet();
1604   intptr_t start = (intptr_t)__ pc();
1605 
1606   // We have received a description of where all the java arg are located
1607   // on entry to the wrapper. We need to convert these args to where
1608   // the jni function will expect them. To figure out where they go
1609   // we convert the java signature to a C signature by inserting
1610   // the hidden arguments as arg[0] and possibly arg[1] (static method)
1611 
1612   const int total_in_args = method->size_of_parameters();
1613   int total_c_args = total_in_args;
1614   if (!is_critical_native) {
1615     total_c_args += 1;
1616     if (method->is_static()) {
1617       total_c_args++;
1618     }
1619   } else {
1620     for (int i = 0; i < total_in_args; i++) {
1621       if (in_sig_bt[i] == T_ARRAY) {
1622         total_c_args++;
1623       }
1624     }
1625   }
1626 
1627   BasicType* out_sig_bt = NEW_RESOURCE_ARRAY(BasicType, total_c_args);
1628   VMRegPair* out_regs   = NEW_RESOURCE_ARRAY(VMRegPair, total_c_args);
1629   BasicType* in_elem_bt = NULL;
1630 
1631   int argc = 0;
1632   if (!is_critical_native) {
1633     out_sig_bt[argc++] = T_ADDRESS;
1634     if (method->is_static()) {
1635       out_sig_bt[argc++] = T_OBJECT;
1636     }
1637 
1638     for (int i = 0; i < total_in_args ; i++ ) {
1639       out_sig_bt[argc++] = in_sig_bt[i];
1640     }
1641   } else {
1642     Thread* THREAD = Thread::current();
1643     in_elem_bt = NEW_RESOURCE_ARRAY(BasicType, total_in_args);
1644     SignatureStream ss(method->signature());
1645     for (int i = 0; i < total_in_args ; i++ ) {
1646       if (in_sig_bt[i] == T_ARRAY) {
1647         // Arrays are passed as int, elem* pair
1648         out_sig_bt[argc++] = T_INT;
1649         out_sig_bt[argc++] = T_ADDRESS;
1650         Symbol* atype = ss.as_symbol(CHECK_NULL);
1651         const char* at = atype->as_C_string();
1652         if (strlen(at) == 2) {
1653           assert(at[0] == '[', "must be");
1654           switch (at[1]) {
1655             case 'B': in_elem_bt[i]  = T_BYTE; break;
1656             case 'C': in_elem_bt[i]  = T_CHAR; break;
1657             case 'D': in_elem_bt[i]  = T_DOUBLE; break;
1658             case 'F': in_elem_bt[i]  = T_FLOAT; break;
1659             case 'I': in_elem_bt[i]  = T_INT; break;
1660             case 'J': in_elem_bt[i]  = T_LONG; break;
1661             case 'S': in_elem_bt[i]  = T_SHORT; break;
1662             case 'Z': in_elem_bt[i]  = T_BOOLEAN; break;
1663             default: ShouldNotReachHere();
1664           }
1665         }
1666       } else {
1667         out_sig_bt[argc++] = in_sig_bt[i];
1668         in_elem_bt[i] = T_VOID;
1669       }
1670       if (in_sig_bt[i] != T_VOID) {
1671         assert(in_sig_bt[i] == ss.type(), "must match");
1672         ss.next();
1673       }
1674     }
1675   }
1676 
1677   // Now figure out where the args must be stored and how much stack space
1678   // they require.
1679   int out_arg_slots;
1680   out_arg_slots = c_calling_convention(out_sig_bt, out_regs, NULL, total_c_args);
1681 
1682   // Compute framesize for the wrapper.  We need to handlize all oops in
1683   // incoming registers
1684 
1685   // Calculate the total number of stack slots we will need.
1686 
1687   // First count the abi requirement plus all of the outgoing args
1688   int stack_slots = SharedRuntime::out_preserve_stack_slots() + out_arg_slots;
1689 
1690   // Now the space for the inbound oop handle area
1691   int total_save_slots = -1;
1692   if (is_critical_native) {
1693     // Critical natives may have to call out so they need a save area
1694     // for register arguments.
1695     int double_slots = 0;
1696     int single_slots = 0;
1697     for ( int i = 0; i < total_in_args; i++) {
1698       if (in_regs[i].first()->is_Register()) {
1699         const Register reg = in_regs[i].first()->as_Register();
1700         switch (in_sig_bt[i]) {
1701           case T_ARRAY:  // critical array (uses 2 slots on LP64)
1702           case T_BOOLEAN:
1703           case T_BYTE:
1704           case T_SHORT:
1705           case T_CHAR:
1706           case T_INT:  single_slots++; break;
1707           case T_LONG: double_slots++; break;
1708           default:  ShouldNotReachHere();
1709         }
1710       } else
1711 #ifdef HARD_FLOAT_CC
1712       if (in_regs[i].first()->is_FloatRegister()) {
1713           switch (in_sig_bt[i]) {
1714               case T_FLOAT:  single_slots++; break;
1715               case T_DOUBLE: double_slots++; break;
1716               default:  ShouldNotReachHere();
1717           }
1718       }
1719 #else
1720       ShouldNotReachHere();
1721 #endif // HARD_FLOAT_CC
1722     }
1723     total_save_slots = double_slots * 2 + single_slots;
1724     // align the save area
1725     if (double_slots != 0) {
1726       stack_slots = round_to(stack_slots, 2);
1727     }
1728   } else {
1729     total_save_slots = 4 * VMRegImpl::slots_per_word;  // 4 arguments passed in registers
1730   }
1731   assert(total_save_slots != -1, "initialize total_save_slots!");
1732 
1733   int oop_handle_offset = stack_slots;
1734   stack_slots += total_save_slots;
1735 
1736   // Now any space we need for handlizing a klass if static method
1737 
1738   int klass_slot_offset = 0;
1739   int klass_offset = -1;
1740   int lock_slot_offset = 0;
1741   bool is_static = false;
1742 
1743   if (method->is_static()) {
1744     klass_slot_offset = stack_slots;
1745     stack_slots += VMRegImpl::slots_per_word;
1746     klass_offset = klass_slot_offset * VMRegImpl::stack_slot_size;
1747     is_static = true;
1748   }
1749 
1750   // Plus a lock if needed
1751 
1752   if (method->is_synchronized()) {
1753     lock_slot_offset = stack_slots;
1754     stack_slots += VMRegImpl::slots_per_word;
1755   }
1756 
1757   // Now a place (+2) to save return values or temp during shuffling
1758   // + 2 for return address (which we own) and saved rfp
1759   stack_slots += 4;
1760 
1761   // Ok The space we have allocated will look like:
1762   //
1763   //
1764   // FP-> | saved lr            |
1765   //      |---------------------|
1766   //      | saved fp            |
1767   //      |---------------------|
1768   //      | 2 slots for moves   |
1769   //      |---------------------|
1770   //      | lock box (if sync)  |
1771   //      |---------------------| <- lock_slot_offset
1772   //      | klass (if static)   |
1773   //      |---------------------| <- klass_slot_offset
1774   //      | oopHandle area      |
1775   //      |---------------------| <- oop_handle_offset (8 java arg registers)
1776   //      | outbound memory     |
1777   //      | based arguments     |
1778   //      |                     |
1779   //      |---------------------|
1780   //      |                     |
1781   // SP-> | out_preserved_slots |
1782   //
1783   //
1784 
1785 
1786   // Now compute actual number of stack words we need rounding to make
1787   // stack properly aligned.
1788   stack_slots = round_to(stack_slots, StackAlignmentInSlots);
1789 
1790   int stack_size = stack_slots * VMRegImpl::stack_slot_size;
1791 
1792   // First thing make an ic check to see if we should even be here
1793 
1794   // We are free to use all registers as temps without saving them and
1795   // restoring them except rfp. rfp is the only callee save register
1796   // as far as the interpreter and the compiler(s) are concerned.
1797 
1798 
1799   const Register ic_reg = rscratch2;
1800   const Register receiver = j_rarg0;
1801 
1802   Label hit;
1803   Label exception_pending;
1804 
1805   assert_different_registers(ic_reg, receiver, rscratch1);
1806   __ verify_oop(receiver);
1807   __ cmp_klass(receiver, ic_reg, rscratch1);
1808   __ b(hit, Assembler::EQ);
1809 
1810   __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1811 
1812   // Verified entry point must be aligned
1813   __ align(8);
1814 
1815   __ bind(hit);
1816 
1817 #ifdef ASSERT
1818   __ mov(ic_reg, 0xdead); // trash ic_reg(rscratch2), as used as real scratch further
1819 #endif
1820 
1821   int vep_offset = ((intptr_t)__ pc()) - start;
1822 
1823   // Generate stack overflow check
1824 
1825   // If we have to make this method not-entrant we'll overwrite its
1826   // first instruction with a jump.  For this action to be legal we
1827   // must ensure that this first instruction is a B, BL, NOP, BKPT,
1828   // SVC, HVC, or SMC.  Make it a NOP.
1829   __ nop();
1830 
1831   if (UseStackBanging) {
1832     __ bang_stack_with_offset(StackShadowPages*os::vm_page_size());
1833   } else {
1834     Unimplemented();
1835   }
1836 
1837   // Generate a new frame for the wrapper.
1838   __ enter();
1839   // -2 because return address is already present and so is saved rfp
1840   __ sub(sp, sp, stack_size - 2*wordSize);
1841 
1842   // Frame is now completed as far as size and linkage.
1843   int frame_complete = ((intptr_t)__ pc()) - start;
1844 
1845   if (is_critical_native) {
1846     check_needs_gc_for_critical_native(masm, stack_slots, total_c_args, total_in_args,
1847                                        oop_handle_offset, oop_maps, in_regs, in_sig_bt);
1848   }
1849 
1850   //
1851   // We immediately shuffle the arguments so that any vm call we have to
1852   // make from here on out (sync slow path, jvmti, etc.) we will have
1853   // captured the oops from our caller and have a valid oopMap for
1854   // them.
1855 
1856   // -----------------
1857   // The Grand Shuffle
1858 
1859   // The Java calling convention is either equal (linux) or denser (win64) than the
1860   // c calling convention. However the because of the jni_env argument the c calling
1861   // convention always has at least one more (and two for static) arguments than Java.
1862   // Therefore if we move the args from java -> c backwards then we will never have
1863   // a register->register conflict and we don't have to build a dependency graph
1864   // and figure out how to break any cycles.
1865   //
1866 
1867   // Record sp-based slot for receiver on stack for non-static methods
1868   int receiver_offset = -1;
1869 
1870   // This is a trick. We double the stack slots so we can claim
1871   // the oops in the caller's frame. Since we are sure to have
1872   // more args than the caller doubling is enough to make
1873   // sure we can capture all the incoming oop args from the
1874   // caller.
1875   //
1876   OopMap* map = new OopMap(stack_slots * 2, 0 /* arg_slots*/);
1877 
1878   // Mark location of rfp (someday)
1879   // map->set_callee_saved(VMRegImpl::stack2reg( stack_slots - 2), stack_slots * 2, 0, vmreg(rfp));
1880 
1881 
1882 #ifdef ASSERT
1883   bool reg_destroyed[RegisterImpl::number_of_registers];
1884   bool freg_destroyed[FloatRegisterImpl::number_of_registers];
1885   for ( int r = 0 ; r < RegisterImpl::number_of_registers ; r++ ) {
1886     reg_destroyed[r] = false;
1887   }
1888   for ( int f = 0 ; f < FloatRegisterImpl::number_of_registers ; f++ ) {
1889     freg_destroyed[f] = false;
1890   }
1891 
1892 #endif // ASSERT
1893 
1894   // This may iterate in two different directions depending on the
1895   // kind of native it is.  The reason is that for regular JNI natives
1896   // the incoming and outgoing registers are offset upwards and for
1897   // critical natives they are offset down.
1898   GrowableArray<int> arg_order(2 * total_in_args);
1899   VMRegPair tmp_vmreg;
1900   tmp_vmreg.set1(rscratch2->as_VMReg());
1901 
1902   if (!is_critical_native) {
1903     for (int i = total_in_args - 1, c_arg = total_c_args - 1; i >= 0; i--, c_arg--) {
1904       arg_order.push(i);
1905       arg_order.push(c_arg);
1906     }
1907   } else {
1908     // Compute a valid move order, using tmp_vmreg to break any cycles
1909     ComputeMoveOrder cmo(total_in_args, in_regs, total_c_args, out_regs, in_sig_bt, arg_order, tmp_vmreg);
1910   }
1911 
1912   int temploc = -1;
1913   for (int ai = 0; ai < arg_order.length(); ai += 2) {
1914     int i = arg_order.at(ai);
1915     int c_arg = arg_order.at(ai + 1);
1916     __ block_comment(err_msg("move %d -> %d", i, c_arg));
1917     if (c_arg == -1) {
1918       assert(is_critical_native, "should only be required for critical natives");
1919       // This arg needs to be moved to a temporary
1920       __ mov(tmp_vmreg.first()->as_Register(), in_regs[i].first()->as_Register());
1921       in_regs[i] = tmp_vmreg;
1922       temploc = i;
1923       continue;
1924     } else if (i == -1) {
1925       assert(is_critical_native, "should only be required for critical natives");
1926       // Read from the temporary location
1927       assert(temploc != -1, "must be valid");
1928       i = temploc;
1929       temploc = -1;
1930     }
1931 #ifdef ASSERT
1932     if (in_regs[i].first()->is_Register()) {
1933       assert(!reg_destroyed[in_regs[i].first()->as_Register()->encoding()], "destroyed reg!");
1934     } else if (in_regs[i].first()->is_FloatRegister()) {
1935       assert(!freg_destroyed[in_regs[i].first()->as_FloatRegister()->encoding()], "destroyed reg!");
1936     }
1937     if (out_regs[c_arg].first()->is_Register()) {
1938       reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1939     } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1940       freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1941     }
1942 #endif // ASSERT
1943     switch (in_sig_bt[i]) {
1944       case T_ARRAY:
1945         if (is_critical_native) {
1946           unpack_array_argument(masm, in_regs[i], in_elem_bt[i], out_regs[c_arg + 1], out_regs[c_arg], rscratch2);
1947           c_arg++;
1948 #ifdef ASSERT
1949           if (out_regs[c_arg].first()->is_Register()) {
1950             reg_destroyed[out_regs[c_arg].first()->as_Register()->encoding()] = true;
1951           } else if (out_regs[c_arg].first()->is_FloatRegister()) {
1952             freg_destroyed[out_regs[c_arg].first()->as_FloatRegister()->encoding()] = true;
1953           }
1954 #endif
1955           break;
1956         }
1957       case T_OBJECT:
1958         assert(!is_critical_native, "no oop arguments");
1959         object_move(masm, map, oop_handle_offset, stack_slots, in_regs[i], out_regs[c_arg],
1960                     ((i == 0) && (!is_static)),
1961                     &receiver_offset);
1962         break;
1963       case T_VOID:
1964         break;
1965 
1966       case T_FLOAT:
1967         float_move(masm, in_regs[i], out_regs[c_arg]);
1968         break;
1969 
1970       case T_DOUBLE:
1971         assert( i + 1 < total_in_args &&
1972                 in_sig_bt[i + 1] == T_VOID &&
1973                 out_sig_bt[c_arg+1] == T_VOID, "bad arg list");
1974         double_move(masm, in_regs[i], out_regs[c_arg]);
1975         break;
1976 
1977       case T_LONG :
1978         long_move(masm, in_regs[i], out_regs[c_arg]);
1979         break;
1980 
1981       case T_BOOLEAN :
1982       case T_BYTE :
1983       case T_CHAR :
1984       case T_SHORT :
1985       case T_INT :
1986         move_int(masm, in_regs[i], out_regs[c_arg]);
1987     break;
1988 
1989       case T_ADDRESS: assert(false, "found T_ADDRESS in java args");
1990       case T_NARROWOOP :
1991       case T_METADATA :
1992       case T_NARROWKLASS :
1993       default:
1994     ShouldNotReachHere();
1995     }
1996   }
1997 
1998   // point c_arg at the first arg that is already loaded in case we
1999   // need to spill before we call out
2000   int c_arg = total_c_args - total_in_args;
2001 
2002   // We use r4 as the oop handle for the receiver/klass
2003   // It is callee save so it survives the call to native
2004 
2005   const Register oop_handle_reg = r4;
2006 
2007   // Pre-load a static method's oop.  Used both by locking code and
2008   // the normal JNI call code.
2009   if (method->is_static() && !is_critical_native) {
2010 
2011     //  load oop into a register
2012     __ movoop(oop_handle_reg,
2013               JNIHandles::make_local(method->method_holder()->java_mirror()),
2014               /*immediate*/true);
2015 
2016     // Now handlize the static class mirror it's known not-null.
2017     __ str(oop_handle_reg, Address(sp, klass_offset));
2018     map->set_oop(VMRegImpl::stack2reg(klass_slot_offset));
2019 
2020     // Now get the handle
2021     __ lea(oop_handle_reg, Address(sp, klass_offset));
2022     // store the klass handle as second argument
2023     __ mov(c_rarg1, oop_handle_reg);
2024     // and protect the arg if we must spill
2025     c_arg--;
2026   }
2027 
2028   // Change state to native (we save the return address in the thread, since it might not
2029   // be pushed on the stack when we do a a stack traversal). It is enough that the pc()
2030   // points into the right code segment. It does not have to be the correct return pc.
2031   // We use the same pc/oopMap repeatedly when we call out
2032 
2033   intptr_t the_pc = (intptr_t) __ pc();
2034   oop_maps->add_gc_map(the_pc - start, map);
2035 
2036   __ set_last_Java_frame(sp, noreg, (address)the_pc, rscratch1);
2037 
2038 
2039   // We have all of the arguments setup at this point. We must not touch any register
2040   // argument registers at this point (what if we save/restore them there are no oop?
2041 
2042 #ifdef DTRACE_ENABLED
2043   {
2044     SkipIfEqual skip(masm, &DTraceMethodProbes, false);
2045     // protect the args we've loaded
2046     (void) save_args(masm, total_c_args, c_arg, out_regs);
2047     __ mov_metadata(c_rarg1, method());
2048     __ call_VM_leaf(
2049       CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_entry),
2050       rthread, c_rarg1);
2051     restore_args(masm, total_c_args, c_arg, out_regs);
2052   }
2053 #endif
2054 
2055   // RedefineClasses() tracing support for obsolete method entry
2056   if (RC_TRACE_IN_RANGE(0x00001000, 0x00002000)) {
2057     // protect the args we've loaded
2058     (void) save_args(masm, total_c_args, c_arg, out_regs);
2059     __ mov_metadata(c_rarg1, method());
2060     __ call_VM_leaf(
2061       CAST_FROM_FN_PTR(address, SharedRuntime::rc_trace_method_entry),
2062       rthread, c_rarg1);
2063     restore_args(masm, total_c_args, c_arg, out_regs);
2064   }
2065 
2066   // Lock a synchronized method
2067 
2068   // Register definitions used by locking and unlocking
2069 
2070   Label slow_path_lock;
2071   Label lock_done;
2072 
2073   if (method->is_synchronized()) {
2074     assert(!is_critical_native, "unhandled");
2075 
2076     // registers below are not used to pass parameters
2077     // and they are caller save in C1
2078     // => safe to use as temporary here
2079 #ifdef COMPILER2
2080     stop("fix temporary register set below");
2081 #endif
2082     const Register swap_reg = r5;
2083     const Register obj_reg  = r6;  // Will contain the oop
2084     const Register lock_reg = r7;  // Address of compiler lock object (BasicLock)
2085 
2086     const int mark_word_offset = BasicLock::displaced_header_offset_in_bytes();
2087 
2088     // Get the handle (the 2nd argument)
2089     __ mov(oop_handle_reg, c_rarg1);
2090 
2091     // Get address of the box
2092 
2093     __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
2094 
2095     // Load the oop from the handle
2096     __ ldr(obj_reg, Address(oop_handle_reg, 0));
2097 
2098     if (UseBiasedLocking) {
2099       __ biased_locking_enter(lock_reg, obj_reg, swap_reg, rscratch2, false, lock_done, &slow_path_lock);
2100     }
2101 
2102     // Load (object->mark() | 1) into swap_reg %r0
2103     __ ldr(swap_reg, Address(obj_reg, 0));
2104     __ orr(swap_reg, swap_reg, 1);
2105 
2106     // Save (object->mark() | 1) into BasicLock's displaced header
2107     __ str(swap_reg, Address(lock_reg, mark_word_offset));
2108 
2109     // src -> dest iff dest == r0 else r0 <- dest
2110     { Label here;
2111       __ cmpxchgptr(swap_reg, lock_reg, obj_reg, rscratch1, lock_done, &slow_path_lock);
2112     }
2113 
2114     // Slow path will re-enter here
2115     __ bind(lock_done);
2116   }
2117 
2118 
2119   // Finally just about ready to make the JNI call
2120 
2121 
2122   // get JNIEnv* which is first argument to native
2123   if (!is_critical_native) {
2124     __ lea(c_rarg0, Address(rthread, in_bytes(JavaThread::jni_environment_offset())));
2125   }
2126 
2127   // Now set thread in native
2128   __ mov(rscratch1, _thread_in_native);
2129   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
2130   __ dmb(Assembler::ISH);
2131   __ str(rscratch1, rscratch2);
2132 
2133   // Do the call
2134   rt_call(masm, native_func);
2135 
2136   // Unpack native results.
2137   switch (ret_type) {
2138   case T_BOOLEAN: __ uxtb(r0, r0);           break;
2139   case T_CHAR   : __ uxth(r0, r0);           break;
2140   case T_BYTE   : __ sxtb(r0, r0);           break;
2141   case T_SHORT  : __ sxth(r0, r0);           break;
2142   case T_INT    :                                    break;
2143   case T_FLOAT  :
2144 #ifndef HARD_FLOAT_CC
2145       if(hasFPU()) {
2146           __ vmov_f32(d0, r0);
2147       }
2148 #endif
2149       break;
2150   case T_DOUBLE :
2151 #ifndef HARD_FLOAT_CC
2152       if(hasFPU()) {
2153           __ vmov_f64(d0, r0, r1);
2154       }
2155 #endif
2156       break;
2157   case T_ARRAY:                 // Really a handle
2158   case T_OBJECT:                // Really a handle
2159       break; // can't de-handlize until after safepoint check
2160   case T_VOID: break;
2161   case T_LONG: break;
2162   default       : ShouldNotReachHere();
2163   }
2164 
2165   // Switch thread to "native transition" state before reading the synchronization state.
2166   // This additional state is necessary because reading and testing the synchronization
2167   // state is not atomic w.r.t. GC, as this scenario demonstrates:
2168   //     Java thread A, in _thread_in_native state, loads _not_synchronized and is preempted.
2169   //     VM thread changes sync state to synchronizing and suspends threads for GC.
2170   //     Thread A is resumed to finish this native method, but doesn't block here since it
2171   //     didn't see any synchronization is progress, and escapes.
2172   __ mov(rscratch1, _thread_in_native_trans);
2173   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
2174   __ dmb(Assembler::ISH);
2175   __ str(rscratch1, rscratch2);
2176 
2177   if(os::is_MP()) {
2178     if (UseMembar) {
2179       // Force this write out before the read below
2180       __ membar(Assembler::AnyAny);
2181     } else {
2182       // Write serialization page so VM thread can do a pseudo remote membar.
2183       // We use the current thread pointer to calculate a thread specific
2184       // offset to write to within the page. This minimizes bus traffic
2185       // due to cache line collision.
2186       __ serialize_memory(rthread, rscratch1);
2187     }
2188   }
2189 
2190   Label after_transition;
2191 
2192   // check for safepoint operation in progress and/or pending suspend requests
2193   {
2194     Label Continue;
2195 
2196     __ mov(rscratch1, ExternalAddress((address)SafepointSynchronize::address_of_state()));
2197     __ ldr(rscratch1, Address(rscratch1));
2198     __ cmp(rscratch1, SafepointSynchronize::_not_synchronized);
2199 
2200     Label L;
2201     __ b(L, Assembler::NE);
2202     __ ldr(rscratch1, Address(rthread, JavaThread::suspend_flags_offset()));
2203     __ cbz(rscratch1, Continue);
2204     __ bind(L);
2205 
2206     // Don't use call_VM as it will see a possible pending exception and forward it
2207     // and never return here preventing us from clearing _last_native_pc down below.
2208     //
2209     save_native_result(masm, ret_type, stack_slots);
2210     __ mov(c_rarg0, rthread);
2211 #ifndef PRODUCT
2212   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
2213 #endif
2214     if (!is_critical_native) {
2215       __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans)));
2216     } else {
2217       __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, JavaThread::check_special_condition_for_native_trans_and_transition)));
2218     }
2219     __ bl(rscratch1);
2220     __ maybe_isb();
2221     // Restore any method result value
2222     restore_native_result(masm, ret_type, stack_slots);
2223 
2224     if (is_critical_native) {
2225       // The call above performed the transition to thread_in_Java so
2226       // skip the transition logic below.
2227       __ b(after_transition);
2228     }
2229 
2230     __ bind(Continue);
2231   }
2232 
2233   // change thread state
2234   __ mov(rscratch1, _thread_in_Java);
2235   __ lea(rscratch2, Address(rthread, JavaThread::thread_state_offset()));
2236   __ dmb(Assembler::ISH);
2237   __ str(rscratch1, rscratch2);
2238   __ bind(after_transition);
2239 
2240   Label reguard;
2241   Label reguard_done;
2242   __ ldrb(rscratch1, Address(rthread, JavaThread::stack_guard_state_offset()));
2243   __ cmp(rscratch1, JavaThread::stack_guard_yellow_disabled);
2244   __ b(reguard, Assembler::EQ);
2245   __ bind(reguard_done);
2246 
2247   // native result if any is live
2248 
2249   // Unlock
2250   Label unlock_done;
2251   Label slow_path_unlock;
2252   if (method->is_synchronized()) {
2253     const Register obj_reg  = r2;  // Will contain the oop
2254     const Register lock_reg = rscratch1; // Address of compiler lock object (BasicLock)
2255     const Register old_hdr  = r3;  // value of old header at unlock time
2256 
2257     // Get locked oop from the handle we passed to jni
2258     __ ldr(obj_reg, Address(oop_handle_reg, 0));
2259 
2260     if (UseBiasedLocking) {
2261       __ biased_locking_exit(obj_reg, old_hdr, unlock_done);
2262     }
2263 
2264     // Simple recursive lock?
2265     // get address of the stack lock
2266     __ lea(lock_reg, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
2267 
2268     //  get old displaced header
2269     __ ldr(old_hdr, Address(lock_reg, 0));
2270     __ cbz(old_hdr, unlock_done);
2271 
2272     // Atomic swap old header if oop still contains the stack lock
2273     Label succeed;
2274     __ cmpxchgptr(lock_reg, old_hdr, obj_reg, rscratch2, succeed, &slow_path_unlock);
2275     __ bind(succeed);
2276 
2277     // slow path re-enters here
2278     __ bind(unlock_done);
2279   }
2280 
2281 #ifdef DTRACE_ENABLED
2282   {
2283     SkipIfEqual skip(masm, &DTraceMethodProbes, false);
2284     save_native_result(masm, ret_type, stack_slots);
2285     __ mov_metadata(c_rarg1, method());
2286     __ call_VM_leaf(
2287          CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit),
2288          rthread, c_rarg1);
2289     restore_native_result(masm, ret_type, stack_slots);
2290   }
2291 #endif
2292 
2293   __ reset_last_Java_frame(false);
2294 
2295   // Unpack oop result
2296   if (ret_type == T_OBJECT || ret_type == T_ARRAY) {
2297     __ resolve_jobject(r0, rthread, rscratch1);
2298   }
2299 
2300   if (!is_critical_native) {
2301     // reset handle block
2302     __ mov(rscratch1, 0);
2303     __ ldr(r2, Address(rthread, JavaThread::active_handles_offset()));
2304     __ str(rscratch1, Address(r2, JNIHandleBlock::top_offset_in_bytes()));
2305   }
2306 
2307   __ leave();
2308 
2309   if (!is_critical_native) {
2310     // Any exception pending?
2311     __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2312     __ cbnz(rscratch1, exception_pending);
2313   }
2314 
2315   // We're done
2316   __ b(lr);
2317 
2318   // Unexpected paths are out of line and go here
2319 
2320   if (!is_critical_native) {
2321     // forward the exception
2322     __ bind(exception_pending);
2323 
2324     // and forward the exception
2325     __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
2326   }
2327 
2328   // Slow path locking & unlocking
2329   if (method->is_synchronized()) {
2330 
2331     // BEGIN Slow path lock
2332     __ bind(slow_path_lock);
2333 
2334     // has last_Java_frame setup. No exceptions so do vanilla call not call_VM
2335     // args are (oop obj, BasicLock* lock, JavaThread* thread)
2336 
2337     // protect the args we've loaded
2338     const int extra_words = save_args(masm, total_c_args, c_arg, out_regs);
2339 
2340     __ ldr(c_rarg0, Address(oop_handle_reg));
2341     __ lea(c_rarg1, Address(sp, (extra_words + lock_slot_offset) * VMRegImpl::stack_slot_size));
2342     __ mov(c_rarg2, rthread);
2343 
2344     // Not a leaf but we have last_Java_frame setup as we want
2345     __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_locking_C), 3);
2346     restore_args(masm, total_c_args, c_arg, out_regs);
2347 
2348 #ifdef ASSERT
2349     { Label L;
2350       __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2351       __ cbz(rscratch1, L);
2352       __ stop("no pending exception allowed on exit from monitorenter");
2353       __ bind(L);
2354     }
2355 #endif
2356     __ b(lock_done);
2357 
2358     // END Slow path lock
2359 
2360     // BEGIN Slow path unlock
2361     __ bind(slow_path_unlock);
2362 
2363     // If we haven't already saved the native result we must save it now as xmm registers
2364     // are still exposed.
2365 
2366     save_native_result(masm, ret_type, stack_slots);
2367 
2368     __ ldr(c_rarg0, Address(oop_handle_reg));
2369     __ lea(c_rarg1, Address(sp, lock_slot_offset * VMRegImpl::stack_slot_size));
2370 
2371     // Save pending exception around call to VM (which contains an EXCEPTION_MARK)
2372     __ ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2373     __ mov(rscratch2, 0);
2374     __ str(rscratch2, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2375 
2376     rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::complete_monitor_unlocking_C));
2377 
2378 #ifdef ASSERT
2379     {
2380       Label L;
2381       __ ldr(rscratch2, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2382       __ cbz(rscratch2, L);
2383       __ stop("no pending exception allowed on exit complete_monitor_unlocking_C");
2384       __ bind(L);
2385     }
2386 #endif // ASSERT
2387 
2388     __ str(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
2389 
2390     restore_native_result(masm, ret_type, stack_slots);
2391 
2392     __ b(unlock_done);
2393 
2394     // END Slow path unlock
2395 
2396   } // synchronized
2397 
2398   // SLOW PATH Reguard the stack if needed
2399 
2400   __ bind(reguard);
2401   save_native_result(masm, ret_type, stack_slots);
2402   rt_call(masm, CAST_FROM_FN_PTR(address, SharedRuntime::reguard_yellow_pages));
2403   restore_native_result(masm, ret_type, stack_slots);
2404   // and continue
2405   __ b(reguard_done);
2406 
2407 
2408 
2409   __ flush();
2410 
2411   nmethod *nm = nmethod::new_native_nmethod(method,
2412                                             compile_id,
2413                                             masm->code(),
2414                                             vep_offset,
2415                                             frame_complete,
2416                                             stack_slots / VMRegImpl::slots_per_word,
2417                                             (is_static ? in_ByteSize(klass_offset) : in_ByteSize(receiver_offset)),
2418                                             in_ByteSize(lock_slot_offset*VMRegImpl::stack_slot_size),
2419                                             oop_maps);
2420 
2421   if (is_critical_native) {
2422     nm->set_lazy_critical_native(true);
2423   }
2424 
2425   return nm;
2426 }
2427 
2428 // this function returns the adjust size (in number of words) to a c2i adapter
2429 // activation for use during deoptimization
2430 int Deoptimization::last_frame_adjust(int callee_parameters, int callee_locals) {
2431   assert(callee_locals >= callee_parameters,
2432           "test and remove; got more parms than locals");
2433   if (callee_locals < callee_parameters)
2434     return 0;                   // No adjustment for negative locals
2435   int diff = (callee_locals - callee_parameters) * Interpreter::stackElementWords;
2436   // diff is counted in stack words
2437   return round_to(diff, 2);
2438 }
2439 
2440 
2441 //------------------------------generate_deopt_blob----------------------------
2442 void SharedRuntime::generate_deopt_blob() {
2443 
2444   // Allocate space for the code
2445   ResourceMark rm;
2446   // Setup code generation tools
2447   CodeBuffer buffer("deopt_blob", 2048, 1024);
2448   MacroAssembler* masm = new MacroAssembler(&buffer);
2449   int frame_size_in_words;
2450   OopMap* map = NULL;
2451   OopMapSet *oop_maps = new OopMapSet();
2452 
2453   // -------------
2454   // This code enters when returning to a de-optimized nmethod.  A return
2455   // address has been pushed on the the stack, and return values are in
2456   // registers.
2457   // If we are doing a normal deopt then we were called from the patched
2458   // nmethod from the point we returned to the nmethod. So the return
2459   // address on the stack is wrong by NativeCall::instruction_size
2460   // We will adjust the value so it looks like we have the original return
2461   // address on the stack (like when we eagerly deoptimized).
2462   // In the case of an exception pending when deoptimizing, we enter
2463   // with a return address on the stack that points after the call we patched
2464   // into the exception handler. We have the following register state from,
2465   // e.g., the forward exception stub (see stubGenerator_x86_64.cpp).
2466   //    r0: exception oop
2467   //    r7: exception handler
2468   //    r3: throwing pc
2469   // So in this case we simply jam r3 into the useless return address and
2470   // the stack looks just like we want.
2471   //
2472   // At this point we need to de-opt.  We save the argument return
2473   // registers.  We call the first C routine, fetch_unroll_info().  This
2474   // routine captures the return values and returns a structure which
2475   // describes the current frame size and the sizes of all replacement frames.
2476   // The current frame is compiled code and may contain many inlined
2477   // functions, each with their own JVM state.  We pop the current frame, then
2478   // push all the new frames.  Then we call the C routine unpack_frames() to
2479   // populate these frames.  Finally unpack_frames() returns us the new target
2480   // address.  Notice that callee-save registers are BLOWN here; they have
2481   // already been captured in the vframeArray at the time the return PC was
2482   // patched.
2483   address start = __ pc();
2484   Label cont;
2485 
2486   // Prolog for non exception case!
2487 
2488   // Save everything in sight.
2489   map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words);
2490 
2491   // Normal deoptimization.  Save exec mode for unpack_frames.
2492   __ mov(r7, Deoptimization::Unpack_deopt); // callee-saved
2493   __ b(cont);
2494 
2495   int reexecute_offset = __ pc() - start;
2496 
2497   // Reexecute case
2498   // return address is the pc describes what bci to do re-execute at
2499 
2500   // No need to update map as each call to save_live_registers will produce identical oopmap
2501   (void) RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words);
2502 
2503   __ mov(r7, Deoptimization::Unpack_reexecute); // callee-saved
2504   __ b(cont);
2505 
2506   int exception_offset = __ pc() - start;
2507 
2508   // Prolog for exception case
2509 
2510   // all registers are dead at this entry point, except for r0, and
2511   // r3 which contain the exception oop and exception pc
2512   // respectively.  Set them in TLS and fall thru to the
2513   // unpack_with_exception_in_tls entry point.
2514 
2515   __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
2516   __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
2517 
2518   int exception_in_tls_offset = __ pc() - start;
2519 
2520   // new implementation because exception oop is now passed in JavaThread
2521 
2522   // Prolog for exception case
2523   // All registers must be preserved because they might be used by LinearScan
2524   // Exceptiop oop and throwing PC are passed in JavaThread
2525   // tos: stack at point of call to method that threw the exception (i.e. only
2526   // args are on the stack, no return address)
2527 
2528   // The return address pushed by save_live_registers will be patched
2529   // later with the throwing pc. The correct value is not available
2530   // now because loading it from memory would destroy registers.
2531 
2532   // NB: The SP at this point must be the SP of the method that is
2533   // being deoptimized.  Deoptimization assumes that the frame created
2534   // here by save_live_registers is immediately below the method's SP.
2535   // This is a somewhat fragile mechanism.
2536 
2537   // Save everything in sight.
2538   map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words);
2539 
2540   // Now it is safe to overwrite any register
2541 
2542   // Deopt during an exception.  Save exec mode for unpack_frames.
2543   __ mov(r7, Deoptimization::Unpack_exception); // callee-saved
2544 
2545   // load throwing pc from JavaThread and patch it as the return address
2546   // of the current frame. Then clear the field in JavaThread
2547 
2548   __ ldr(r3, Address(rthread, JavaThread::exception_pc_offset()));
2549   __ str(r3, Address(rfp));
2550   __ mov(rscratch1, 0);
2551   __ str(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
2552 
2553 #ifdef ASSERT
2554   // verify that there is really an exception oop in JavaThread
2555   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
2556   __ verify_oop(r0);
2557 
2558   // verify that there is no pending exception
2559   Label no_pending_exception;
2560   __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
2561   __ cbz(rscratch1, no_pending_exception);
2562   __ stop("must not have pending exception here");
2563   __ bind(no_pending_exception);
2564 #endif
2565 
2566   __ bind(cont);
2567 
2568   // Call C code.  Need thread and this frame, but NOT official VM entry
2569   // crud.  We cannot block on this call, no GC can happen.
2570   //
2571   // UnrollBlock* fetch_unroll_info(JavaThread* thread)
2572 
2573   // fetch_unroll_info needs to call last_java_frame().
2574 
2575   Label retaddr;
2576   __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2577 #ifdef ASSERT0
2578   { Label L;
2579     __ ldr(rscratch1, Address(rthread,
2580                               JavaThread::last_Java_fp_offset()));
2581     __ cbz(rscratch1, L);
2582     __ stop("SharedRuntime::generate_deopt_blob: last_Java_fp not cleared");
2583     __ bind(L);
2584   }
2585 #endif // ASSERT
2586   __ mov(c_rarg0, rthread);
2587   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::fetch_unroll_info)));
2588   __ bl(rscratch1);
2589   __ bind(retaddr);
2590 
2591   // Need to have an oopmap that tells fetch_unroll_info where to
2592   // find any register it might need.
2593   oop_maps->add_gc_map(__ pc() - start, map);
2594 
2595   __ reset_last_Java_frame(false);
2596 
2597   // Load UnrollBlock* into rdi
2598   __ mov(r5, r0);
2599 
2600    Label noException;
2601   __ cmp(r7, Deoptimization::Unpack_exception);   // Was exception pending?
2602   __ b(noException, Assembler::NE);
2603   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
2604   // QQQ this is useless it was NULL above
2605   __ ldr(r3, Address(rthread, JavaThread::exception_pc_offset()));
2606   __ mov(rscratch1, 0);
2607   __ str(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
2608   __ str(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
2609 
2610   __ verify_oop(r0);
2611 
2612   // Overwrite the result registers with the exception results.
2613   __ str(r0, Address(sp, RegisterSaver::offset_in_bytes(RegisterSaver::r0_off)));
2614   // I think this is useless
2615   // __ str(r3, Address(sp, RegisterSaver::r3_offset_in_bytes()));
2616 
2617   __ bind(noException);
2618 
2619   // Only register save data is on the stack.
2620   // Now restore the result registers.  Everything else is either dead
2621   // or captured in the vframeArray.
2622   RegisterSaver::restore_result_registers(masm);
2623 
2624   // All of the register save area has been popped of the stack. Only the
2625   // return address remains.
2626 
2627   // Pop all the frames we must move/replace.
2628   //
2629   // Frame picture (youngest to oldest)
2630   // 1: self-frame (no frame link)
2631   // 2: deopting frame  (no frame link)
2632   // 3: caller of deopting frame (could be compiled/interpreted).
2633   //
2634   // Note: by leaving the return address of self-frame on the stack
2635   // and using the size of frame 2 to adjust the stack
2636   // when we are done the return to frame 3 will still be on the stack.
2637 
2638   // Pop deoptimized frame
2639   __ ldr(r2, Address(r5, Deoptimization::UnrollBlock::size_of_deoptimized_frame_offset_in_bytes()));
2640   __ sub(r2, r2, 2 * wordSize);
2641   __ add(sp, sp, r2);
2642   __ ldrd(rfp, lr, __ post(sp, 2 * wordSize));
2643   // LR should now be the return address to the caller (3)
2644 
2645 #ifdef ASSERT
2646   // Compilers generate code that bang the stack by as much as the
2647   // interpreter would need. So this stack banging should never
2648   // trigger a fault. Verify that it does not on non product builds.
2649   if (UseStackBanging) {
2650     __ ldr(rscratch2, Address(r5, Deoptimization::UnrollBlock::total_frame_sizes_offset_in_bytes()));
2651     __ bang_stack_size(rscratch2, r2);
2652   }
2653 #endif
2654   // Load address of array of frame pcs into r2
2655   __ ldr(r2, Address(r5, Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
2656 
2657   // Trash the old pc
2658   // __ addptr(sp, wordSize);  FIXME ????
2659 
2660   // Load address of array of frame sizes into r4
2661   __ ldr(r4, Address(r5, Deoptimization::UnrollBlock::frame_sizes_offset_in_bytes()));
2662 
2663   // Load counter into r3
2664   __ ldr(r3, Address(r5, Deoptimization::UnrollBlock::number_of_frames_offset_in_bytes()));
2665 
2666   // Now adjust the caller's stack to make up for the extra locals
2667   // but record the original sp so that we can save it in the skeletal interpreter
2668   // frame and the stack walking of interpreter_sender will get the unextended sp
2669   // value and not the "real" sp value.
2670 
2671   const Register sender_sp = r6;
2672 
2673   __ mov(sender_sp, sp);
2674   __ ldr(rscratch1, Address(r5,
2675                        Deoptimization::UnrollBlock::
2676                        caller_adjustment_offset_in_bytes()));
2677   __ sub(sp, sp, rscratch1);
2678 
2679   // Push interpreter frames in a loop
2680   __ mov(rscratch1, (address)0xDEADDEAD);        // Make a recognizable pattern
2681   // Initially used to place 0xDEADDEAD in rscratch2 as well - why?
2682   __ mov(rscratch2, 0);
2683   Label loop;
2684   __ bind(loop);
2685   __ ldr(rscratch1, Address(__ post(r4, wordSize)));          // Load frame size
2686   __ sub(rscratch1, rscratch1, 2*wordSize);           // We'll push pc and fp by hand
2687   __ ldr(lr, Address(__ post(r2, wordSize)));  // Load pc
2688   __ enter();                           // Save old & set new fp
2689   __ sub(sp, sp, rscratch1);                  // Prolog
2690   // This value is corrected by layout_activation_impl
2691   __ str(rscratch2, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize));
2692   __ str(sender_sp, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize)); // Make it walkable
2693   __ mov(sender_sp, sp);               // Pass sender_sp to next frame
2694   __ sub(r3, r3, 1);                   // Decrement counter
2695   __ cbnz(r3, loop);
2696 
2697     // Re-push self-frame
2698   __ ldr(lr, Address(r2));
2699   __ enter();
2700 
2701   // Allocate a full sized register save area.  We subtract 2 because
2702   // enter() just pushed 2 words
2703   __ sub(sp, sp, (frame_size_in_words - 2) * wordSize);
2704 
2705   // Restore frame locals after moving the frame
2706   if(hasFPU()) {
2707     __ vstr_f64(d0, Address(sp, RegisterSaver::offset_in_bytes(RegisterSaver::fpu_state_off)));
2708   }
2709   __ strd(r0, Address(sp, RegisterSaver::offset_in_bytes(RegisterSaver::r0_off)));
2710 
2711   // Call C code.  Need thread but NOT official VM entry
2712   // crud.  We cannot block on this call, no GC can happen.  Call should
2713   // restore return values to their stack-slots with the new SP.
2714   //
2715   // void Deoptimization::unpack_frames(JavaThread* thread, int exec_mode)
2716 
2717   // Use rfp because the frames look interpreted now
2718   // Don't need the precise return PC here, just precise enough to point into this code blob.
2719   address the_pc = __ pc();
2720   __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
2721 
2722   __ mov(c_rarg0, rthread);
2723   __ mov(c_rarg1, r7); // second arg: exec_mode
2724   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
2725   __ bl(rscratch1);
2726 
2727   // Set an oopmap for the call site
2728   // Use the same PC we used for the last java frame
2729   oop_maps->add_gc_map(the_pc - start,
2730                        new OopMap( frame_size_in_words, 0 ));
2731 
2732   // Clear fp AND pc
2733   __ reset_last_Java_frame(true);
2734 
2735   // Collect return values
2736   if(hasFPU()) {
2737     __ vldr_f64(d0, Address(sp, RegisterSaver::offset_in_bytes(RegisterSaver::fpu_state_off)));
2738   }
2739   __ ldrd(r0, Address(sp, RegisterSaver::offset_in_bytes(RegisterSaver::r0_off)));
2740   // I think this is useless (throwing pc?)
2741   // __ ldr(r3, Address(sp, RegisterSaver::r3_offset_in_bytes()));
2742 
2743   // Pop self-frame.
2744   __ leave();                           // Epilog
2745 
2746   // Jump to interpreter
2747   __ b(lr);
2748 
2749   // Make sure all code is generated
2750   masm->flush();
2751 
2752   _deopt_blob = DeoptimizationBlob::create(&buffer, oop_maps, 0, exception_offset, reexecute_offset, frame_size_in_words);
2753   _deopt_blob->set_unpack_with_exception_in_tls_offset(exception_in_tls_offset);
2754 
2755 }
2756 
2757 uint SharedRuntime::out_preserve_stack_slots() {
2758   return 0;
2759 }
2760 
2761 #ifdef COMPILER2
2762 //------------------------------generate_uncommon_trap_blob--------------------
2763 /*void SharedRuntime::generate_uncommon_trap_blob() {
2764   // Allocate space for the code
2765   ResourceMark rm;
2766   // Setup code generation tools
2767   CodeBuffer buffer("uncommon_trap_blob", 2048, 1024);
2768   MacroAssembler* masm = new MacroAssembler(&buffer);
2769 
2770   assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
2771 
2772   address start = __ pc();
2773 
2774   // Push self-frame.  We get here with a return address in LR
2775   // and sp should be 16 byte aligned
2776   // push rfp and retaddr by hand
2777   __ strd(rfp, lr, Address(__ pre(sp, -2 * wordSize)));
2778   // we don't expect an arg reg save area
2779 #ifndef PRODUCT
2780   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
2781 #endif
2782   // compiler left unloaded_class_index in j_rarg0 move to where the
2783   // runtime expects it.
2784   if (c_rarg1 != j_rarg0) {
2785     __ mov(c_rarg1, j_rarg0);
2786   }
2787 
2788   // we need to set the past SP to the stack pointer of the stub frame
2789   // and the pc to the address where this runtime call will return
2790   // although actually any pc in this code blob will do).
2791   Label retaddr;
2792   __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2793 
2794   // Call C code.  Need thread but NOT official VM entry
2795   // crud.  We cannot block on this call, no GC can happen.  Call should
2796   // capture callee-saved registers as well as return values.
2797   // Thread is in rdi already.
2798   //
2799   // UnrollBlock* uncommon_trap(JavaThread* thread, jint unloaded_class_index);
2800   //
2801   // n.b. 2 gp args, 0 fp args, integral return type
2802 
2803   __ mov(c_rarg0, rthread);
2804   __ lea(rscratch1,
2805          RuntimeAddress(CAST_FROM_FN_PTR(address,
2806                                          Deoptimization::uncommon_trap)));
2807   __ bl(rscratch1);
2808   __ bind(retaddr);
2809 
2810   // Set an oopmap for the call site
2811   OopMapSet* oop_maps = new OopMapSet();
2812   OopMap* map = new OopMap(SimpleRuntimeFrame::framesize, 0);
2813 
2814   // location of rfp is known implicitly by the frame sender code
2815 
2816   oop_maps->add_gc_map(__ pc() - start, map);
2817 
2818   __ reset_last_Java_frame(false);
2819 
2820   // move UnrollBlock* into r4
2821   __ mov(r4, r0);
2822 
2823   // Pop all the frames we must move/replace.
2824   //
2825   // Frame picture (youngest to oldest)
2826   // 1: self-frame (no frame link)
2827   // 2: deopting frame  (no frame link)
2828   // 3: caller of deopting frame (could be compiled/interpreted).
2829 
2830   // Pop self-frame.  We have no frame, and must rely only on r0 and sp.
2831   __ add(sp, sp, (SimpleRuntimeFrame::framesize) << LogBytesPerInt); // Epilog!
2832 
2833   // Pop deoptimized frame (int)
2834   __ ldr(r2, Address(r4,
2835                      Deoptimization::UnrollBlock::
2836                      size_of_deoptimized_frame_offset_in_bytes()));
2837   __ sub(r2, r2, 2 * wordSize);
2838   __ add(sp, sp, r2);
2839   __ ldrd(rfp, lr, __ post(sp, 2 * wordSize));
2840   // LR should now be the return address to the caller (3) frame
2841 
2842 #ifdef ASSERT
2843   // Compilers generate code that bang the stack by as much as the
2844   // interpreter would need. So this stack banging should never
2845   // trigger a fault. Verify that it does not on non product builds.
2846   if (UseStackBanging) {
2847     __ ldr(r1, Address(r4,
2848                        Deoptimization::UnrollBlock::
2849                        total_frame_sizes_offset_in_bytes()));
2850     __ bang_stack_size(r1, r2);
2851   }
2852 #endif
2853 
2854   // Load address of array of frame pcs into r2 (address*)
2855   __ ldr(r2, Address(r4,
2856                      Deoptimization::UnrollBlock::frame_pcs_offset_in_bytes()));
2857 
2858   // Load address of array of frame sizes into r5 (intptr_t*)
2859   __ ldr(r5, Address(r4,
2860                      Deoptimization::UnrollBlock::
2861                      frame_sizes_offset_in_bytes()));
2862 
2863   // Counter
2864   __ ldr(r3, Address(r4,
2865                      Deoptimization::UnrollBlock::
2866                      number_of_frames_offset_in_bytes())); // (int)
2867 
2868   // Now adjust the caller's stack to make up for the extra locals but
2869   // record the original sp so that we can save it in the skeletal
2870   // interpreter frame and the stack walking of interpreter_sender
2871   // will get the unextended sp value and not the "real" sp value.
2872 
2873   const Register sender_sp = r8;
2874 
2875   __ mov(sender_sp, sp);
2876   __ ldr(r1, Address(r4,
2877                      Deoptimization::UnrollBlock::
2878                      caller_adjustment_offset_in_bytes())); // (int)
2879   __ sub(sp, sp, r1);
2880 
2881   __ mov(rscratch1, 0);
2882   // Push interpreter frames in a loop
2883   Label loop;
2884   __ bind(loop);
2885   __ ldr(r1, Address(r5, 0));       // Load frame size
2886   __ sub(r1, r1, 2 * wordSize);     // We'll push pc and rfp by hand
2887   __ ldr(lr, Address(r2, 0));       // Save return address
2888   __ enter();                       // and old rfp & set new rfp
2889   __ sub(sp, sp, r1);               // Prolog
2890   __ str(sender_sp, Address(rfp, frame::interpreter_frame_sender_sp_offset * wordSize)); // Make it walkable
2891   // This value is corrected by layout_activation_impl
2892   __ str(rscratch1, Address(rfp, frame::interpreter_frame_last_sp_offset * wordSize)); //zero it
2893   __ mov(sender_sp, sp);          // Pass sender_sp to next frame
2894   __ add(r5, r5, wordSize);       // Bump array pointer (sizes)
2895   __ add(r2, r2, wordSize);       // Bump array pointer (pcs)
2896   __ subs(r3, r3, 1);             // Decrement counter
2897   __ b(loop, Assembler::GT);
2898   __ ldr(lr, Address(r2, 0));     // save final return address
2899   // Re-push self-frame
2900   __ enter();                     // & old rfp & set new rfp
2901 
2902   // Use rfp because the frames look interpreted now
2903   // Save "the_pc" since it cannot easily be retrieved using the last_java_SP after we aligned SP.
2904   // Don't need the precise return PC here, just precise enough to point into this code blob.
2905   address the_pc = __ pc();
2906   __ set_last_Java_frame(sp, rfp, the_pc, rscratch1);
2907 
2908   // Call C code.  Need thread but NOT official VM entry
2909   // crud.  We cannot block on this call, no GC can happen.  Call should
2910   // restore return values to their stack-slots with the new SP.
2911   // Thread is in rdi already.
2912   //
2913   // BasicType unpack_frames(JavaThread* thread, int exec_mode);
2914   //
2915   // n.b. 2 gp args, 0 fp args, integral return type
2916 
2917   // sp should already be aligned
2918   __ mov(c_rarg0, rthread);
2919   __ mov(c_rarg1, (unsigned)Deoptimization::Unpack_uncommon_trap);
2920   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, Deoptimization::unpack_frames)));
2921   __ bl(rscratch1);
2922 
2923   // Set an oopmap for the call site
2924   // Use the same PC we used for the last java frame
2925   oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0));
2926 
2927   // Clear fp AND pc
2928   __ reset_last_Java_frame(true);
2929 
2930   // Pop self-frame.
2931   __ leave();                 // Epilog
2932 
2933   // Jump to interpreter
2934   __ b(lr);
2935 
2936   // Make sure all code is generated
2937   masm->flush();
2938 
2939   _uncommon_trap_blob =  UncommonTrapBlob::create(&buffer, oop_maps,
2940                                                  SimpleRuntimeFrame::framesize >> 1);
2941 
2942 } */
2943 #endif // COMPILER2
2944 
2945 
2946 //------------------------------generate_handler_blob------
2947 //
2948 // Generate a special Compile2Runtime blob that saves all registers,
2949 // and setup oopmap.
2950 //
2951 SafepointBlob* SharedRuntime::generate_handler_blob(address call_ptr, int poll_type) {
2952   ResourceMark rm;
2953   OopMapSet *oop_maps = new OopMapSet();
2954   OopMap* map;
2955 
2956   // Allocate space for the code.  Setup code generation tools.
2957   CodeBuffer buffer("handler_blob", 2048, 1024);
2958   MacroAssembler* masm = new MacroAssembler(&buffer);
2959 
2960   address start   = __ pc();
2961   address call_pc = NULL;
2962   int frame_size_in_words;
2963   bool cause_return = (poll_type == POLL_AT_RETURN);
2964   bool save_vectors = (poll_type == POLL_AT_VECTOR_LOOP);
2965 
2966   // Save registers, fpu state, and flags
2967   map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words);
2968 
2969   // The following is basically a call_VM.  However, we need the precise
2970   // address of the call in order to generate an oopmap. Hence, we do all the
2971   // work outselves.
2972 
2973   Label retaddr;
2974   __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
2975 
2976   // The return address must always be correct so that frame constructor never
2977   // sees an invalid pc.
2978 
2979   if (!cause_return) {
2980     // overwrite the return address pushed by save_live_registers
2981     __ ldr(lr, Address(rthread, JavaThread::saved_exception_pc_offset()));
2982     __ str(lr, Address(rfp));
2983   }
2984 
2985   // Do the call
2986   __ mov(c_rarg0, rthread);
2987   __ lea(rscratch1, RuntimeAddress(call_ptr));
2988   __ bl(rscratch1);
2989   __ bind(retaddr);
2990 
2991   // Set an oopmap for the call site.  This oopmap will map all
2992   // oop-registers and debug-info registers as callee-saved.  This
2993   // will allow deoptimization at this safepoint to find all possible
2994   // debug-info recordings, as well as let GC find all oops.
2995 
2996   oop_maps->add_gc_map( __ pc() - start, map);
2997 
2998   Label noException;
2999 
3000   __ reset_last_Java_frame(false);
3001 
3002   __ maybe_isb();
3003   __ membar(Assembler::LoadLoad | Assembler::LoadStore);
3004 
3005   __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
3006   __ cbz(rscratch1, noException);
3007 
3008   // Exception pending
3009 
3010   RegisterSaver::restore_live_registers(masm);
3011 
3012   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3013 
3014   // No exception case
3015   __ bind(noException);
3016 
3017   // Normal exit, restore registers and exit.
3018   RegisterSaver::restore_live_registers(masm);
3019 
3020   __ b(lr);
3021 
3022   // Make sure all code is generated
3023   masm->flush();
3024 
3025   // Fill-out other meta info
3026   return SafepointBlob::create(&buffer, oop_maps, frame_size_in_words);
3027 }
3028 
3029 //
3030 // generate_resolve_blob - call resolution (static/virtual/opt-virtual/ic-miss
3031 //
3032 // Generate a stub that calls into vm to find out the proper destination
3033 // of a java call. All the argument registers are live at this point
3034 // but since this is generic code we don't know what they are and the caller
3035 // must do any gc of the args.
3036 //
3037 RuntimeStub* SharedRuntime::generate_resolve_blob(address destination, const char* name) {
3038   assert (StubRoutines::forward_exception_entry() != NULL, "must be generated before");
3039 
3040   // allocate space for the code
3041   ResourceMark rm;
3042 
3043   //CodeBuffer buffer(name, 1000, 512);
3044   CodeBuffer buffer(name, 2048, 512 ); // changed as error later
3045   MacroAssembler* masm                = new MacroAssembler(&buffer);
3046 
3047   int frame_size_in_words;
3048 
3049   OopMapSet *oop_maps = new OopMapSet();
3050   OopMap* map = NULL;
3051 
3052   int start = __ offset();
3053 
3054   map = RegisterSaver::save_live_registers(masm, 0, &frame_size_in_words);
3055 
3056   int frame_complete = __ offset();
3057 
3058   {
3059     Label retaddr;
3060     __ set_last_Java_frame(sp, noreg, retaddr, rscratch1);
3061 
3062     __ mov(c_rarg0, rthread);
3063     __ lea(rscratch1, RuntimeAddress(destination));
3064 
3065     __ bl(rscratch1);
3066     __ bind(retaddr);
3067   }
3068 
3069   // Set an oopmap for the call site.
3070   // We need this not only for callee-saved registers, but also for volatile
3071   // registers that the compiler might be keeping live across a safepoint.
3072 
3073   oop_maps->add_gc_map( __ offset() - start, map);
3074 
3075   __ maybe_isb();
3076 
3077   // r0 contains the address we are going to jump to assuming no exception got installed
3078 
3079   // clear last_Java_sp
3080   // TODO x86 have different action: reset_last_Java_frame(thread, true(fp));
3081   // TODO below is false(fp)
3082   __ reset_last_Java_frame(false);
3083   // check for pending exceptions
3084   Label pending;
3085   __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
3086   __ cbnz(rscratch1, pending);
3087 
3088   // get the returned Method*
3089   __ get_vm_result_2(rmethod, rthread);
3090   __ str(rmethod, Address(sp, RegisterSaver::offset_in_bytes(RegisterSaver::rmethod_off)));
3091 
3092   // r0 is where we want to jump, overwrite rscratch1 which is saved and scratch
3093   __ str(r0, Address(sp, RegisterSaver::offset_in_bytes(RegisterSaver::rscratch1_off)));
3094   RegisterSaver::restore_live_registers(masm);
3095 
3096   // We are back the the original state on entry and ready to go.
3097 
3098   __ b(rscratch1);
3099 
3100   // Pending exception after the safepoint
3101 
3102   __ bind(pending);
3103 
3104   RegisterSaver::restore_live_registers(masm);
3105 
3106   // exception pending => remove activation and forward to exception handler
3107   __ mov(rscratch1, 0);
3108   __ str(rscratch1, Address(rthread, JavaThread::vm_result_offset()));
3109 
3110   __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
3111   __ far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
3112 
3113   // -------------
3114   // make sure all code is generated
3115   masm->flush();
3116 
3117   // return the  blob
3118   // frame_size_words or bytes??
3119   return RuntimeStub::new_runtime_stub(name, &buffer, frame_complete, frame_size_in_words, oop_maps, true);
3120 }
3121 
3122 
3123 #ifdef COMPILER2
3124 // This is here instead of runtime_x86_64.cpp because it uses SimpleRuntimeFrame
3125 //
3126 //------------------------------generate_exception_blob---------------------------
3127 // creates exception blob at the end
3128 // Using exception blob, this code is jumped from a compiled method.
3129 // (see emit_exception_handler in x86_64.ad file)
3130 //
3131 // Given an exception pc at a call we call into the runtime for the
3132 // handler in this method. This handler might merely restore state
3133 // (i.e. callee save registers) unwind the frame and jump to the
3134 // exception handler for the nmethod if there is no Java level handler
3135 // for the nmethod.
3136 //
3137 // This code is entered with a jmp.
3138 //
3139 // Arguments:
3140 //   r0: exception oop
3141 //   r3: exception pc
3142 //
3143 // Results:
3144 //   r0: exception oop
3145 //   r3: exception pc in caller or ???
3146 //   destination: exception handler of caller
3147 //
3148 // Note: the exception pc MUST be at a call (precise debug information)
3149 //       Registers r0, r3, r2, r4, r5, r8-r11 are not callee saved.
3150 //
3151 
3152 void OptoRuntime::generate_exception_blob() {
3153   assert(!OptoRuntime::is_callee_saved_register(R3_num), "");
3154   assert(!OptoRuntime::is_callee_saved_register(R0_num), "");
3155   assert(!OptoRuntime::is_callee_saved_register(R2_num), "");
3156 
3157   assert(SimpleRuntimeFrame::framesize % 4 == 0, "sp not 16-byte aligned");
3158 
3159   // Allocate space for the code
3160   ResourceMark rm;
3161   // Setup code generation tools
3162   CodeBuffer buffer("exception_blob", 2048, 1024);
3163   MacroAssembler* masm = new MacroAssembler(&buffer);
3164 
3165   __ stop("FIXME generate_exception_blob");
3166   // TODO check various assumptions made here
3167   //
3168   // make sure we do so before running this
3169 
3170   address start = __ pc();
3171 
3172   // push rfp and retaddr by hand
3173   // Exception pc is 'return address' for stack walker
3174   __ strd(rfp, lr, Address(__ pre(sp, -2 * wordSize)));
3175   // there are no callee save registers and we don't expect an
3176   // arg reg save area
3177 #ifndef PRODUCT
3178   assert(frame::arg_reg_save_area_bytes == 0, "not expecting frame reg save area");
3179 #endif
3180   // Store exception in Thread object. We cannot pass any arguments to the
3181   // handle_exception call, since we do not want to make any assumption
3182   // about the size of the frame where the exception happened in.
3183   __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
3184   __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
3185 
3186   // This call does all the hard work.  It checks if an exception handler
3187   // exists in the method.
3188   // If so, it returns the handler address.
3189   // If not, it prepares for stack-unwinding, restoring the callee-save
3190   // registers of the frame being removed.
3191   //
3192   // address OptoRuntime::handle_exception_C(JavaThread* thread)
3193   //
3194   // n.b. 1 gp arg, 0 fp args, integral return type
3195 
3196   // the stack should always be aligned
3197   address the_pc = __ pc();
3198   __ set_last_Java_frame(sp, noreg, the_pc, rscratch1);
3199   __ mov(c_rarg0, rthread);
3200   __ lea(rscratch1, RuntimeAddress(CAST_FROM_FN_PTR(address, OptoRuntime::handle_exception_C)));
3201   __ bl(rscratch1);
3202   __ maybe_isb();
3203 
3204   // Set an oopmap for the call site.  This oopmap will only be used if we
3205   // are unwinding the stack.  Hence, all locations will be dead.
3206   // Callee-saved registers will be the same as the frame above (i.e.,
3207   // handle_exception_stub), since they were restored when we got the
3208   // exception.
3209 
3210   OopMapSet* oop_maps = new OopMapSet();
3211 
3212   oop_maps->add_gc_map(the_pc - start, new OopMap(SimpleRuntimeFrame::framesize, 0));
3213 
3214   __ reset_last_Java_frame(false);
3215 
3216   // Restore callee-saved registers
3217 
3218   // rfp is an implicitly saved callee saved register (i.e. the calling
3219   // convention will save restore it in prolog/epilog) Other than that
3220   // there are no callee save registers now that adapter frames are gone.
3221   // and we dont' expect an arg reg save area
3222   __ ldrd(rfp, r3, Address(__ post(sp, 2 * wordSize)));
3223 
3224   // r0: exception handler
3225 
3226   // We have a handler in r0 (could be deopt blob).
3227   __ mov(r8, r0);
3228 
3229   // Get the exception oop
3230   __ ldr(r0, Address(rthread, JavaThread::exception_oop_offset()));
3231   // Get the exception pc in case we are deoptimized
3232   __ ldr(r4, Address(rthread, JavaThread::exception_pc_offset()));
3233   __ mov(rscratch1, 0);
3234 #ifdef ASSERT
3235   __ str(rscratch1, Address(rthread, JavaThread::exception_handler_pc_offset()));
3236   __ str(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
3237 #endif
3238   // Clear the exception oop so GC no longer processes it as a root.
3239   __ str(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
3240 
3241   // r0: exception oop
3242   // r8:  exception handler
3243   // r4: exception pc
3244   // Jump to handler
3245 
3246   __ b(r8);
3247 
3248   // Make sure all code is generated
3249   masm->flush();
3250 
3251   // Set exception blob
3252   _exception_blob =  ExceptionBlob::create(&buffer, oop_maps, SimpleRuntimeFrame::framesize >> 1);
3253 }
3254 #endif // COMPILER2