1 /*
   2  * Copyright (c) 2013, Red Hat Inc.
   3  * Copyright (c) 1999, 2011, Oracle and/or its affiliates.
   4  * All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Defs.hpp"
  31 #include "c1/c1_MacroAssembler.hpp"
  32 #include "c1/c1_Runtime1.hpp"
  33 #include "compiler/disassembler.hpp"
  34 #include "interpreter/interpreter.hpp"
  35 #include "nativeInst_aarch64.hpp"
  36 #include "oops/compiledICHolder.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "prims/jvmtiExport.hpp"
  39 #include "register_aarch64.hpp"
  40 #include "runtime/sharedRuntime.hpp"
  41 #include "runtime/signature.hpp"
  42 #include "runtime/vframe.hpp"
  43 #include "runtime/vframeArray.hpp"
  44 #include "vmreg_aarch64.inline.hpp"
  45 #if INCLUDE_ALL_GCS
  46 #include "gc_implementation/shenandoah/shenandoahBarrierSet.hpp"
  47 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
  48 #endif
  49 
  50 
  51 // Implementation of StubAssembler
  52 
  53 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
  54   // setup registers
  55   assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
  56   assert(oop_result1 != rthread && metadata_result != rthread, "registers must be different");
  57   assert(args_size >= 0, "illegal args_size");
  58   bool align_stack = false;
  59 
  60   mov(c_rarg0, rthread);
  61   set_num_rt_args(0); // Nothing on stack
  62 
  63   Label retaddr;
  64   set_last_Java_frame(sp, rfp, retaddr, rscratch1);
  65 
  66   // do the call
  67   lea(rscratch1, RuntimeAddress(entry));
  68   blr(rscratch1);
  69   bind(retaddr);
  70   int call_offset = offset();
  71   // verify callee-saved register
  72 #ifdef ASSERT
  73   push(r0, sp);
  74   { Label L;
  75     get_thread(r0);
  76     cmp(rthread, r0);
  77     br(Assembler::EQ, L);
  78     stop("StubAssembler::call_RT: rthread not callee saved?");
  79     bind(L);
  80   }
  81   pop(r0, sp);
  82 #endif
  83   reset_last_Java_frame(true);
  84   maybe_isb();
  85 
  86   // check for pending exceptions
  87   { Label L;
  88     // check for pending exceptions (java_thread is set upon return)
  89     ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
  90     cbz(rscratch1, L);
  91     // exception pending => remove activation and forward to exception handler
  92     // make sure that the vm_results are cleared
  93     if (oop_result1->is_valid()) {
  94       str(zr, Address(rthread, JavaThread::vm_result_offset()));
  95     }
  96     if (metadata_result->is_valid()) {
  97       str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
  98     }
  99     if (frame_size() == no_frame_size) {
 100       leave();
 101       far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 102     } else if (_stub_id == Runtime1::forward_exception_id) {
 103       should_not_reach_here();
 104     } else {
 105       far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
 106     }
 107     bind(L);
 108   }
 109   // get oop results if there are any and reset the values in the thread
 110   if (oop_result1->is_valid()) {
 111     get_vm_result(oop_result1, rthread);
 112   }
 113   if (metadata_result->is_valid()) {
 114     get_vm_result_2(metadata_result, rthread);
 115   }
 116   return call_offset;
 117 }
 118 
 119 
 120 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {
 121   mov(c_rarg1, arg1);
 122   return call_RT(oop_result1, metadata_result, entry, 1);
 123 }
 124 
 125 
 126 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {
 127   if (c_rarg1 == arg2) {
 128     if (c_rarg2 == arg1) {
 129       mov(rscratch1, arg1);
 130       mov(arg1, arg2);
 131       mov(arg2, rscratch1);
 132     } else {
 133       mov(c_rarg2, arg2);
 134       mov(c_rarg1, arg1);
 135     }
 136   } else {
 137     mov(c_rarg1, arg1);
 138     mov(c_rarg2, arg2);
 139   }
 140   return call_RT(oop_result1, metadata_result, entry, 2);
 141 }
 142 
 143 
 144 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
 145   // if there is any conflict use the stack
 146   if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
 147       arg2 == c_rarg1 || arg1 == c_rarg3 ||
 148       arg3 == c_rarg1 || arg1 == c_rarg2) {
 149     stp(arg3, arg2, Address(pre(sp, 2 * wordSize)));
 150     stp(arg1, zr, Address(pre(sp, -2 * wordSize)));
 151     ldp(c_rarg1, zr, Address(post(sp, 2 * wordSize)));
 152     ldp(c_rarg3, c_rarg2, Address(post(sp, 2 * wordSize)));
 153   } else {
 154     mov(c_rarg1, arg1);
 155     mov(c_rarg2, arg2);
 156     mov(c_rarg3, arg3);
 157   }
 158   return call_RT(oop_result1, metadata_result, entry, 3);
 159 }
 160 
 161 // Implementation of StubFrame
 162 
 163 class StubFrame: public StackObj {
 164  private:
 165   StubAssembler* _sasm;
 166 
 167  public:
 168   StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments);
 169   void load_argument(int offset_in_words, Register reg);
 170 
 171   ~StubFrame();
 172 };;
 173 
 174 
 175 #define __ _sasm->
 176 
 177 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) {
 178   _sasm = sasm;
 179   __ set_info(name, must_gc_arguments);
 180   __ enter();
 181 }
 182 
 183 // load parameters that were stored with LIR_Assembler::store_parameter
 184 // Note: offsets for store_parameter and load_argument must match
 185 void StubFrame::load_argument(int offset_in_words, Register reg) {
 186   // rbp, + 0: link
 187   //     + 1: return address
 188   //     + 2: argument with offset 0
 189   //     + 3: argument with offset 1
 190   //     + 4: ...
 191 
 192   __ ldr(reg, Address(rfp, (offset_in_words + 2) * BytesPerWord));
 193 }
 194 
 195 
 196 StubFrame::~StubFrame() {
 197   __ leave();
 198   __ ret(lr);
 199 }
 200 
 201 #undef __
 202 
 203 
 204 // Implementation of Runtime1
 205 
 206 #define __ sasm->
 207 
 208 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
 209 
 210 // Stack layout for saving/restoring  all the registers needed during a runtime
 211 // call (this includes deoptimization)
 212 // Note: note that users of this frame may well have arguments to some runtime
 213 // while these values are on the stack. These positions neglect those arguments
 214 // but the code in save_live_registers will take the argument count into
 215 // account.
 216 //
 217 
 218 enum reg_save_layout {
 219   reg_save_frame_size = 32 /* float */ + 32 /* integer */
 220 };
 221 
 222 // Save off registers which might be killed by calls into the runtime.
 223 // Tries to smart of about FP registers.  In particular we separate
 224 // saving and describing the FPU registers for deoptimization since we
 225 // have to save the FPU registers twice if we describe them.  The
 226 // deopt blob is the only thing which needs to describe FPU registers.
 227 // In all other cases it should be sufficient to simply save their
 228 // current value.
 229 
 230 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs];
 231 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs];
 232 static int reg_save_size_in_words;
 233 static int frame_size_in_bytes = -1;
 234 
 235 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {
 236   int frame_size_in_bytes = reg_save_frame_size * BytesPerWord;
 237   sasm->set_frame_size(frame_size_in_bytes / BytesPerWord);
 238   int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);
 239   OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
 240 
 241   for (int i = 0; i < FrameMap::nof_cpu_regs; i++) {
 242     Register r = as_Register(i);
 243     if (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding()) {
 244       int sp_offset = cpu_reg_save_offsets[i];
 245       oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
 246                                 r->as_VMReg());
 247     }
 248   }
 249 
 250   if (save_fpu_registers) {
 251     for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
 252       FloatRegister r = as_FloatRegister(i);
 253       {
 254         int sp_offset = fpu_reg_save_offsets[i];
 255         oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
 256                                   r->as_VMReg());
 257       }
 258     }
 259   }
 260   return oop_map;
 261 }
 262 
 263 static OopMap* save_live_registers(StubAssembler* sasm,
 264                                    bool save_fpu_registers = true) {
 265   __ block_comment("save_live_registers");
 266 
 267   __ push(RegSet::range(r0, r29), sp);         // integer registers except lr & sp
 268 
 269   if (save_fpu_registers) {
 270     for (int i = 30; i >= 0; i -= 2)
 271       __ stpd(as_FloatRegister(i), as_FloatRegister(i+1),
 272               Address(__ pre(sp, -2 * wordSize)));
 273   } else {
 274     __ add(sp, sp, -32 * wordSize);
 275   }
 276 
 277   return generate_oop_map(sasm, save_fpu_registers);
 278 }
 279 
 280 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
 281   if (restore_fpu_registers) {
 282     for (int i = 0; i < 32; i += 2)
 283       __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
 284               Address(__ post(sp, 2 * wordSize)));
 285   } else {
 286     __ add(sp, sp, 32 * wordSize);
 287   }
 288 
 289   __ pop(RegSet::range(r0, r29), sp);
 290 }
 291 
 292 static void restore_live_registers_except_r0(StubAssembler* sasm, bool restore_fpu_registers = true)  {
 293 
 294   if (restore_fpu_registers) {
 295     for (int i = 0; i < 32; i += 2)
 296       __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
 297               Address(__ post(sp, 2 * wordSize)));
 298   } else {
 299     __ add(sp, sp, 32 * wordSize);
 300   }
 301 
 302   __ ldp(zr, r1, Address(__ post(sp, 16)));
 303   __ pop(RegSet::range(r2, r29), sp);
 304 }
 305 
 306 
 307 
 308 void Runtime1::initialize_pd() {
 309   int i;
 310   int sp_offset = 0;
 311 
 312   // all float registers are saved explicitly
 313   assert(FrameMap::nof_fpu_regs == 32, "double registers not handled here");
 314   for (i = 0; i < FrameMap::nof_fpu_regs; i++) {
 315     fpu_reg_save_offsets[i] = sp_offset;
 316     sp_offset += 2;   // SP offsets are in halfwords
 317   }
 318 
 319   for (i = 0; i < FrameMap::nof_cpu_regs; i++) {
 320     Register r = as_Register(i);
 321     cpu_reg_save_offsets[i] = sp_offset;
 322     sp_offset += 2;   // SP offsets are in halfwords
 323   }
 324 }
 325 
 326 
 327 // target: the entry point of the method that creates and posts the exception oop
 328 // has_argument: true if the exception needs an argument (passed in rscratch1)
 329 
 330 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
 331   // make a frame and preserve the caller's caller-save registers
 332   OopMap* oop_map = save_live_registers(sasm);
 333   int call_offset;
 334   if (!has_argument) {
 335     call_offset = __ call_RT(noreg, noreg, target);
 336   } else {
 337     call_offset = __ call_RT(noreg, noreg, target, rscratch1);
 338   }
 339   OopMapSet* oop_maps = new OopMapSet();
 340   oop_maps->add_gc_map(call_offset, oop_map);
 341 
 342   __ should_not_reach_here();
 343   return oop_maps;
 344 }
 345 
 346 
 347 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
 348   __ block_comment("generate_handle_exception");
 349 
 350   // incoming parameters
 351   const Register exception_oop = r0;
 352   const Register exception_pc  = r3;
 353   // other registers used in this stub
 354 
 355   // Save registers, if required.
 356   OopMapSet* oop_maps = new OopMapSet();
 357   OopMap* oop_map = NULL;
 358   switch (id) {
 359   case forward_exception_id:
 360     // We're handling an exception in the context of a compiled frame.
 361     // The registers have been saved in the standard places.  Perform
 362     // an exception lookup in the caller and dispatch to the handler
 363     // if found.  Otherwise unwind and dispatch to the callers
 364     // exception handler.
 365     oop_map = generate_oop_map(sasm, 1 /*thread*/);
 366 
 367     // load and clear pending exception oop into r0
 368     __ ldr(exception_oop, Address(rthread, Thread::pending_exception_offset()));
 369     __ str(zr, Address(rthread, Thread::pending_exception_offset()));
 370 
 371     // load issuing PC (the return address for this stub) into r3
 372     __ ldr(exception_pc, Address(rfp, 1*BytesPerWord));
 373 
 374     // make sure that the vm_results are cleared (may be unnecessary)
 375     __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
 376     __ str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
 377     break;
 378   case handle_exception_nofpu_id:
 379   case handle_exception_id:
 380     // At this point all registers MAY be live.
 381     oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id);
 382     break;
 383   case handle_exception_from_callee_id: {
 384     // At this point all registers except exception oop (r0) and
 385     // exception pc (lr) are dead.
 386     const int frame_size = 2 /*fp, return address*/;
 387     oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
 388     sasm->set_frame_size(frame_size);
 389     break;
 390   }
 391   default:
 392     __ should_not_reach_here();
 393     break;
 394   }
 395 
 396   // verify that only r0 and r3 are valid at this time
 397   __ invalidate_registers(false, true, true, false, true, true);
 398   // verify that r0 contains a valid exception
 399   __ verify_not_null_oop(exception_oop);
 400 
 401 #ifdef ASSERT
 402   // check that fields in JavaThread for exception oop and issuing pc are
 403   // empty before writing to them
 404   Label oop_empty;
 405   __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 406   __ cbz(rscratch1, oop_empty);
 407   __ stop("exception oop already set");
 408   __ bind(oop_empty);
 409 
 410   Label pc_empty;
 411   __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 412   __ cbz(rscratch1, pc_empty);
 413   __ stop("exception pc already set");
 414   __ bind(pc_empty);
 415 #endif
 416 
 417   // save exception oop and issuing pc into JavaThread
 418   // (exception handler will load it from here)
 419   __ str(exception_oop, Address(rthread, JavaThread::exception_oop_offset()));
 420   __ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset()));
 421 
 422   // patch throwing pc into return address (has bci & oop map)
 423   __ str(exception_pc, Address(rfp, 1*BytesPerWord));
 424 
 425   // compute the exception handler.
 426   // the exception oop and the throwing pc are read from the fields in JavaThread
 427   int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
 428   oop_maps->add_gc_map(call_offset, oop_map);
 429 
 430   // r0: handler address
 431   //      will be the deopt blob if nmethod was deoptimized while we looked up
 432   //      handler regardless of whether handler existed in the nmethod.
 433 
 434   // only r0 is valid at this time, all other registers have been destroyed by the runtime call
 435   __ invalidate_registers(false, true, true, true, true, true);
 436 
 437   // patch the return address, this stub will directly return to the exception handler
 438   __ str(r0, Address(rfp, 1*BytesPerWord));
 439 
 440   switch (id) {
 441   case forward_exception_id:
 442   case handle_exception_nofpu_id:
 443   case handle_exception_id:
 444     // Restore the registers that were saved at the beginning.
 445     restore_live_registers(sasm, id != handle_exception_nofpu_id);
 446     break;
 447   case handle_exception_from_callee_id:
 448     // WIN64_ONLY: No need to add frame::arg_reg_save_area_bytes to SP
 449     // since we do a leave anyway.
 450 
 451     // Pop the return address since we are possibly changing SP (restoring from BP).
 452     __ leave();
 453 
 454     // Restore SP from FP if the exception PC is a method handle call site.
 455     {
 456       Label nope;
 457       __ ldrw(rscratch1, Address(rthread, JavaThread::is_method_handle_return_offset()));
 458       __ cbzw(rscratch1, nope);
 459       __ mov(sp, rfp);
 460       __ bind(nope);
 461     }
 462 
 463     __ ret(lr);  // jump to exception handler
 464     break;
 465   default:  ShouldNotReachHere();
 466   }
 467 
 468   return oop_maps;
 469 }
 470 
 471 
 472 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
 473   // incoming parameters
 474   const Register exception_oop = r0;
 475   // callee-saved copy of exception_oop during runtime call
 476   const Register exception_oop_callee_saved = r19;
 477   // other registers used in this stub
 478   const Register exception_pc = r3;
 479   const Register handler_addr = r1;
 480 
 481   // verify that only r0, is valid at this time
 482   __ invalidate_registers(false, true, true, true, true, true);
 483 
 484 #ifdef ASSERT
 485   // check that fields in JavaThread for exception oop and issuing pc are empty
 486   Label oop_empty;
 487   __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 488   __ cbz(rscratch1, oop_empty);
 489   __ stop("exception oop must be empty");
 490   __ bind(oop_empty);
 491 
 492   Label pc_empty;
 493   __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 494   __ cbz(rscratch1, pc_empty);
 495   __ stop("exception pc must be empty");
 496   __ bind(pc_empty);
 497 #endif
 498 
 499   // Save our return address because
 500   // exception_handler_for_return_address will destroy it.  We also
 501   // save exception_oop
 502   __ stp(lr, exception_oop, Address(__ pre(sp, -2 * wordSize)));
 503 
 504   // search the exception handler address of the caller (using the return address)
 505   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, lr);
 506   // r0: exception handler address of the caller
 507 
 508   // Only R0 is valid at this time; all other registers have been
 509   // destroyed by the call.
 510   __ invalidate_registers(false, true, true, true, false, true);
 511 
 512   // move result of call into correct register
 513   __ mov(handler_addr, r0);
 514 
 515   // get throwing pc (= return address).
 516   // lr has been destroyed by the call
 517   __ ldp(lr, exception_oop, Address(__ post(sp, 2 * wordSize)));
 518   __ mov(r3, lr);
 519 
 520   __ verify_not_null_oop(exception_oop);
 521 
 522   {
 523     Label foo;
 524     __ ldrw(rscratch1, Address(rthread, JavaThread::is_method_handle_return_offset()));
 525     __ cbzw(rscratch1, foo);
 526     __ mov(sp, rfp);
 527     __ bind(foo);
 528   }
 529 
 530   // continue at exception handler (return address removed)
 531   // note: do *not* remove arguments when unwinding the
 532   //       activation since the caller assumes having
 533   //       all arguments on the stack when entering the
 534   //       runtime to determine the exception handler
 535   //       (GC happens at call site with arguments!)
 536   // r0: exception oop
 537   // r3: throwing pc
 538   // r1: exception handler
 539   __ br(handler_addr);
 540 }
 541 
 542 
 543 
 544 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
 545   // use the maximum number of runtime-arguments here because it is difficult to
 546   // distinguish each RT-Call.
 547   // Note: This number affects also the RT-Call in generate_handle_exception because
 548   //       the oop-map is shared for all calls.
 549   DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
 550   assert(deopt_blob != NULL, "deoptimization blob must have been created");
 551 
 552   OopMap* oop_map = save_live_registers(sasm);
 553 
 554   __ mov(c_rarg0, rthread);
 555   Label retaddr;
 556   __ set_last_Java_frame(sp, rfp, retaddr, rscratch1);
 557   // do the call
 558   __ lea(rscratch1, RuntimeAddress(target));
 559   __ blr(rscratch1);
 560   __ bind(retaddr);
 561   OopMapSet* oop_maps = new OopMapSet();
 562   oop_maps->add_gc_map(__ offset(), oop_map);
 563   // verify callee-saved register
 564 #ifdef ASSERT
 565   { Label L;
 566     __ get_thread(rscratch1);
 567     __ cmp(rthread, rscratch1);
 568     __ br(Assembler::EQ, L);
 569     __ stop("StubAssembler::call_RT: rthread not callee saved?");
 570     __ bind(L);
 571   }
 572 #endif
 573   __ reset_last_Java_frame(true);
 574   __ maybe_isb();
 575 
 576   // check for pending exceptions
 577   { Label L;
 578     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 579     __ cbz(rscratch1, L);
 580     // exception pending => remove activation and forward to exception handler
 581 
 582     { Label L1;
 583       __ cbnz(r0, L1);                                  // have we deoptimized?
 584       __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
 585       __ bind(L1);
 586     }
 587 
 588     // the deopt blob expects exceptions in the special fields of
 589     // JavaThread, so copy and clear pending exception.
 590 
 591     // load and clear pending exception
 592     __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
 593     __ str(zr, Address(rthread, Thread::pending_exception_offset()));
 594 
 595     // check that there is really a valid exception
 596     __ verify_not_null_oop(r0);
 597 
 598     // load throwing pc: this is the return address of the stub
 599     __ mov(r3, lr);
 600 
 601 #ifdef ASSERT
 602     // check that fields in JavaThread for exception oop and issuing pc are empty
 603     Label oop_empty;
 604     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 605     __ cbz(rscratch1, oop_empty);
 606     __ stop("exception oop must be empty");
 607     __ bind(oop_empty);
 608 
 609     Label pc_empty;
 610     __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 611     __ cbz(rscratch1, pc_empty);
 612     __ stop("exception pc must be empty");
 613     __ bind(pc_empty);
 614 #endif
 615 
 616     // store exception oop and throwing pc to JavaThread
 617     __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
 618     __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
 619 
 620     restore_live_registers(sasm);
 621 
 622     __ leave();
 623 
 624     // Forward the exception directly to deopt blob. We can blow no
 625     // registers and must leave throwing pc on the stack.  A patch may
 626     // have values live in registers so the entry point with the
 627     // exception in tls.
 628     __ far_jump(RuntimeAddress(deopt_blob->unpack_with_exception_in_tls()));
 629 
 630     __ bind(L);
 631   }
 632 
 633 
 634   // Runtime will return true if the nmethod has been deoptimized during
 635   // the patching process. In that case we must do a deopt reexecute instead.
 636 
 637   Label reexecuteEntry, cont;
 638 
 639   __ cbz(r0, cont);                                 // have we deoptimized?
 640 
 641   // Will reexecute. Proper return address is already on the stack we just restore
 642   // registers, pop all of our frame but the return address and jump to the deopt blob
 643   restore_live_registers(sasm);
 644   __ leave();
 645   __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
 646 
 647   __ bind(cont);
 648   restore_live_registers(sasm);
 649   __ leave();
 650   __ ret(lr);
 651 
 652   return oop_maps;
 653 }
 654 
 655 
 656 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
 657 
 658   const Register exception_oop = r0;
 659   const Register exception_pc  = r3;
 660 
 661   // for better readability
 662   const bool must_gc_arguments = true;
 663   const bool dont_gc_arguments = false;
 664 
 665   // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
 666   bool save_fpu_registers = true;
 667 
 668   // stub code & info for the different stubs
 669   OopMapSet* oop_maps = NULL;
 670   OopMap* oop_map = NULL;
 671   switch (id) {
 672     {
 673     case forward_exception_id:
 674       {
 675         oop_maps = generate_handle_exception(id, sasm);
 676         __ leave();
 677         __ ret(lr);
 678       }
 679       break;
 680 
 681     case throw_div0_exception_id:
 682       { StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments);
 683         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
 684       }
 685       break;
 686 
 687     case throw_null_pointer_exception_id:
 688       { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments);
 689         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
 690       }
 691       break;
 692 
 693     case new_instance_id:
 694     case fast_new_instance_id:
 695     case fast_new_instance_init_check_id:
 696       {
 697         Register klass = r3; // Incoming
 698         Register obj   = r0; // Result
 699 
 700         if (id == new_instance_id) {
 701           __ set_info("new_instance", dont_gc_arguments);
 702         } else if (id == fast_new_instance_id) {
 703           __ set_info("fast new_instance", dont_gc_arguments);
 704         } else {
 705           assert(id == fast_new_instance_init_check_id, "bad StubID");
 706           __ set_info("fast new_instance init check", dont_gc_arguments);
 707         }
 708 
 709         if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
 710             UseTLAB && FastTLABRefill) {
 711           Label slow_path;
 712           Register obj_size = r2;
 713           Register t1       = r19;
 714           Register t2       = r4;
 715           assert_different_registers(klass, obj, obj_size, t1, t2);
 716 
 717           __ stp(r5, r19, Address(__ pre(sp, -2 * wordSize)));
 718 
 719           if (id == fast_new_instance_init_check_id) {
 720             // make sure the klass is initialized
 721             __ ldrb(rscratch1, Address(klass, InstanceKlass::init_state_offset()));
 722             __ cmpw(rscratch1, InstanceKlass::fully_initialized);
 723             __ br(Assembler::NE, slow_path);
 724           }
 725 
 726 #ifdef ASSERT
 727           // assert object can be fast path allocated
 728           {
 729             Label ok, not_ok;
 730             __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 731             __ cmp(obj_size, 0u);
 732             __ br(Assembler::LE, not_ok);  // make sure it's an instance (LH > 0)
 733             __ tstw(obj_size, Klass::_lh_instance_slow_path_bit);
 734             __ br(Assembler::EQ, ok);
 735             __ bind(not_ok);
 736             __ stop("assert(can be fast path allocated)");
 737             __ should_not_reach_here();
 738             __ bind(ok);
 739           }
 740 #endif // ASSERT
 741 
 742           // if we got here then the TLAB allocation failed, so try
 743           // refilling the TLAB or allocating directly from eden.
 744           Label retry_tlab, try_eden;
 745           __ tlab_refill(retry_tlab, try_eden, slow_path); // does not destroy r3 (klass), returns r5
 746 
 747           __ bind(retry_tlab);
 748 
 749           // get the instance size (size is postive so movl is fine for 64bit)
 750           __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 751 
 752           __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path);
 753 
 754           __ initialize_object(obj, klass, obj_size, 0, t1, t2);
 755           __ verify_oop(obj);
 756           __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
 757           __ ret(lr);
 758 
 759           __ bind(try_eden);
 760           // get the instance size (size is postive so movl is fine for 64bit)
 761           __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 762 
 763           __ eden_allocate(obj, obj_size, 0, t1, slow_path);
 764           __ incr_allocated_bytes(rthread, obj_size, 0, rscratch1);
 765 
 766           __ initialize_object(obj, klass, obj_size, 0, t1, t2);
 767           __ verify_oop(obj);
 768           __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
 769           __ ret(lr);
 770 
 771           __ bind(slow_path);
 772           __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
 773         }
 774 
 775         __ enter();
 776         OopMap* map = save_live_registers(sasm);
 777         int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
 778         oop_maps = new OopMapSet();
 779         oop_maps->add_gc_map(call_offset, map);
 780         restore_live_registers_except_r0(sasm);
 781         __ verify_oop(obj);
 782         __ leave();
 783         __ ret(lr);
 784 
 785         // r0,: new instance
 786       }
 787 
 788       break;
 789 
 790     case counter_overflow_id:
 791       {
 792         Register bci = r0, method = r1;
 793         __ enter();
 794         OopMap* map = save_live_registers(sasm);
 795         // Retrieve bci
 796         __ ldrw(bci, Address(rfp, 2*BytesPerWord));
 797         // And a pointer to the Method*
 798         __ ldr(method, Address(rfp, 3*BytesPerWord));
 799         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
 800         oop_maps = new OopMapSet();
 801         oop_maps->add_gc_map(call_offset, map);
 802         restore_live_registers(sasm);
 803         __ leave();
 804         __ ret(lr);
 805       }
 806       break;
 807 
 808     case new_type_array_id:
 809     case new_object_array_id:
 810       {
 811         Register length   = r19; // Incoming
 812         Register klass    = r3; // Incoming
 813         Register obj      = r0; // Result
 814 
 815         if (id == new_type_array_id) {
 816           __ set_info("new_type_array", dont_gc_arguments);
 817         } else {
 818           __ set_info("new_object_array", dont_gc_arguments);
 819         }
 820 
 821 #ifdef ASSERT
 822         // assert object type is really an array of the proper kind
 823         {
 824           Label ok;
 825           Register t0 = obj;
 826           __ ldrw(t0, Address(klass, Klass::layout_helper_offset()));
 827           __ asrw(t0, t0, Klass::_lh_array_tag_shift);
 828           int tag = ((id == new_type_array_id)
 829                      ? Klass::_lh_array_tag_type_value
 830                      : Klass::_lh_array_tag_obj_value);
 831           __ mov(rscratch1, tag);
 832           __ cmpw(t0, rscratch1);
 833           __ br(Assembler::EQ, ok);
 834           __ stop("assert(is an array klass)");
 835           __ should_not_reach_here();
 836           __ bind(ok);
 837         }
 838 #endif // ASSERT
 839 
 840         if (UseTLAB && FastTLABRefill) {
 841           Register arr_size = r4;
 842           Register t1       = r2;
 843           Register t2       = r5;
 844           Label slow_path;
 845           assert_different_registers(length, klass, obj, arr_size, t1, t2);
 846 
 847           // check that array length is small enough for fast path.
 848           __ mov(rscratch1, C1_MacroAssembler::max_array_allocation_length);
 849           __ cmpw(length, rscratch1);
 850           __ br(Assembler::HI, slow_path);
 851 
 852           // if we got here then the TLAB allocation failed, so try
 853           // refilling the TLAB or allocating directly from eden.
 854           Label retry_tlab, try_eden;
 855           const Register thread =
 856             __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves r19 & r3, returns rthread
 857 
 858           __ bind(retry_tlab);
 859 
 860           // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
 861           // since size is positive ldrw does right thing on 64bit
 862           __ ldrw(t1, Address(klass, Klass::layout_helper_offset()));
 863           __ lslvw(arr_size, length, t1);
 864           __ ubfx(t1, t1, Klass::_lh_header_size_shift,
 865                   exact_log2(Klass::_lh_header_size_mask + 1));
 866           __ add(arr_size, arr_size, t1);
 867           __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
 868           __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
 869 
 870           __ tlab_allocate(obj, arr_size, 0, t1, t2, slow_path);  // preserves arr_size
 871 
 872           __ initialize_header(obj, klass, length, t1, t2);
 873           __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte)));
 874           assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
 875           assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
 876           __ andr(t1, t1, Klass::_lh_header_size_mask);
 877           __ sub(arr_size, arr_size, t1);  // body length
 878           __ add(t1, t1, obj);       // body start
 879           __ initialize_body(t1, arr_size, 0, t2);
 880           __ membar(Assembler::StoreStore);
 881           __ verify_oop(obj);
 882 
 883           __ ret(lr);
 884 
 885           __ bind(try_eden);
 886           // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
 887           // since size is positive ldrw does right thing on 64bit
 888           __ ldrw(t1, Address(klass, Klass::layout_helper_offset()));
 889           // since size is postive movw does right thing on 64bit
 890           __ movw(arr_size, length);
 891           __ lslvw(arr_size, length, t1);
 892           __ ubfx(t1, t1, Klass::_lh_header_size_shift,
 893                   exact_log2(Klass::_lh_header_size_mask + 1));
 894           __ add(arr_size, arr_size, t1);
 895           __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
 896           __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
 897 
 898           __ eden_allocate(obj, arr_size, 0, t1, slow_path);  // preserves arr_size
 899           __ incr_allocated_bytes(thread, arr_size, 0, rscratch1);
 900 
 901           __ initialize_header(obj, klass, length, t1, t2);
 902           __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte)));
 903           assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
 904           assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
 905           __ andr(t1, t1, Klass::_lh_header_size_mask);
 906           __ sub(arr_size, arr_size, t1);  // body length
 907           __ add(t1, t1, obj);       // body start
 908           __ initialize_body(t1, arr_size, 0, t2);
 909           __ membar(Assembler::StoreStore);
 910           __ verify_oop(obj);
 911 
 912           __ ret(lr);
 913 
 914           __ bind(slow_path);
 915         }
 916 
 917         __ enter();
 918         OopMap* map = save_live_registers(sasm);
 919         int call_offset;
 920         if (id == new_type_array_id) {
 921           call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
 922         } else {
 923           call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
 924         }
 925 
 926         oop_maps = new OopMapSet();
 927         oop_maps->add_gc_map(call_offset, map);
 928         restore_live_registers_except_r0(sasm);
 929 
 930         __ verify_oop(obj);
 931         __ leave();
 932         __ ret(lr);
 933 
 934         // r0: new array
 935       }
 936       break;
 937 
 938     case new_multi_array_id:
 939       { StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
 940         // r0,: klass
 941         // r19,: rank
 942         // r2: address of 1st dimension
 943         OopMap* map = save_live_registers(sasm);
 944         __ mov(c_rarg1, r0);
 945         __ mov(c_rarg3, r2);
 946         __ mov(c_rarg2, r19);
 947         int call_offset = __ call_RT(r0, noreg, CAST_FROM_FN_PTR(address, new_multi_array), r1, r2, r3);
 948 
 949         oop_maps = new OopMapSet();
 950         oop_maps->add_gc_map(call_offset, map);
 951         restore_live_registers_except_r0(sasm);
 952 
 953         // r0,: new multi array
 954         __ verify_oop(r0);
 955       }
 956       break;
 957 
 958     case register_finalizer_id:
 959       {
 960         __ set_info("register_finalizer", dont_gc_arguments);
 961 
 962         // This is called via call_runtime so the arguments
 963         // will be place in C abi locations
 964 
 965         __ verify_oop(c_rarg0);
 966 
 967         // load the klass and check the has finalizer flag
 968         Label register_finalizer;
 969         Register t = r5;
 970         __ load_klass(t, r0);
 971         __ ldrw(t, Address(t, Klass::access_flags_offset()));
 972         __ tst(t, JVM_ACC_HAS_FINALIZER);
 973         __ br(Assembler::NE, register_finalizer);
 974         __ ret(lr);
 975 
 976         __ bind(register_finalizer);
 977         __ enter();
 978         OopMap* oop_map = save_live_registers(sasm);
 979         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), r0);
 980         oop_maps = new OopMapSet();
 981         oop_maps->add_gc_map(call_offset, oop_map);
 982 
 983         // Now restore all the live registers
 984         restore_live_registers(sasm);
 985 
 986         __ leave();
 987         __ ret(lr);
 988       }
 989       break;
 990 
 991     case throw_class_cast_exception_id:
 992       { StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments);
 993         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
 994       }
 995       break;
 996 
 997     case throw_incompatible_class_change_error_id:
 998       { StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments);
 999         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
1000       }
1001       break;
1002 
1003     case slow_subtype_check_id:
1004       {
1005         // Typical calling sequence:
1006         // __ push(klass_RInfo);  // object klass or other subclass
1007         // __ push(sup_k_RInfo);  // array element klass or other superclass
1008         // __ bl(slow_subtype_check);
1009         // Note that the subclass is pushed first, and is therefore deepest.
1010         enum layout {
1011           r0_off, r0_off_hi,
1012           r2_off, r2_off_hi,
1013           r4_off, r4_off_hi,
1014           r5_off, r5_off_hi,
1015           sup_k_off, sup_k_off_hi,
1016           klass_off, klass_off_hi,
1017           framesize,
1018           result_off = sup_k_off
1019         };
1020 
1021         __ set_info("slow_subtype_check", dont_gc_arguments);
1022         __ push(RegSet::of(r0, r2, r4, r5), sp);
1023 
1024         // This is called by pushing args and not with C abi
1025         // __ ldr(r4, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // subclass
1026         // __ ldr(r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // superclass
1027 
1028         __ ldp(r4, r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size));
1029 
1030         Label miss;
1031         __ check_klass_subtype_slow_path(r4, r0, r2, r5, NULL, &miss);
1032 
1033         // fallthrough on success:
1034         __ mov(rscratch1, 1);
1035         __ str(rscratch1, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
1036         __ pop(RegSet::of(r0, r2, r4, r5), sp);
1037         __ ret(lr);
1038 
1039         __ bind(miss);
1040         __ str(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
1041         __ pop(RegSet::of(r0, r2, r4, r5), sp);
1042         __ ret(lr);
1043       }
1044       break;
1045 
1046     case monitorenter_nofpu_id:
1047       save_fpu_registers = false;
1048       // fall through
1049     case monitorenter_id:
1050       {
1051         StubFrame f(sasm, "monitorenter", dont_gc_arguments);
1052         OopMap* map = save_live_registers(sasm, save_fpu_registers);
1053 
1054         // Called with store_parameter and not C abi
1055 
1056         f.load_argument(1, r0); // r0,: object
1057         f.load_argument(0, r1); // r1,: lock address
1058 
1059         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), r0, r1);
1060 
1061         oop_maps = new OopMapSet();
1062         oop_maps->add_gc_map(call_offset, map);
1063         restore_live_registers(sasm, save_fpu_registers);
1064       }
1065       break;
1066 
1067     case monitorexit_nofpu_id:
1068       save_fpu_registers = false;
1069       // fall through
1070     case monitorexit_id:
1071       {
1072         StubFrame f(sasm, "monitorexit", dont_gc_arguments);
1073         OopMap* map = save_live_registers(sasm, save_fpu_registers);
1074 
1075         // Called with store_parameter and not C abi
1076 
1077         f.load_argument(0, r0); // r0,: lock address
1078 
1079         // note: really a leaf routine but must setup last java sp
1080         //       => use call_RT for now (speed can be improved by
1081         //       doing last java sp setup manually)
1082         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), r0);
1083 
1084         oop_maps = new OopMapSet();
1085         oop_maps->add_gc_map(call_offset, map);
1086         restore_live_registers(sasm, save_fpu_registers);
1087       }
1088       break;
1089 
1090     case deoptimize_id:
1091       {
1092         StubFrame f(sasm, "deoptimize", dont_gc_arguments);
1093         OopMap* oop_map = save_live_registers(sasm);
1094         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize));
1095         oop_maps = new OopMapSet();
1096         oop_maps->add_gc_map(call_offset, oop_map);
1097         restore_live_registers(sasm);
1098         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1099         assert(deopt_blob != NULL, "deoptimization blob must have been created");
1100         __ leave();
1101         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1102       }
1103       break;
1104 
1105     case throw_range_check_failed_id:
1106       { StubFrame f(sasm, "range_check_failed", dont_gc_arguments);
1107         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
1108       }
1109       break;
1110 
1111     case unwind_exception_id:
1112       { __ set_info("unwind_exception", dont_gc_arguments);
1113         // note: no stubframe since we are about to leave the current
1114         //       activation and we are calling a leaf VM function only.
1115         generate_unwind_exception(sasm);
1116       }
1117       break;
1118 
1119     case access_field_patching_id:
1120       { StubFrame f(sasm, "access_field_patching", dont_gc_arguments);
1121         // we should set up register map
1122         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
1123       }
1124       break;
1125 
1126     case load_klass_patching_id:
1127       { StubFrame f(sasm, "load_klass_patching", dont_gc_arguments);
1128         // we should set up register map
1129         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
1130       }
1131       break;
1132 
1133     case load_mirror_patching_id:
1134       { StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments);
1135         // we should set up register map
1136         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
1137       }
1138       break;
1139 
1140     case load_appendix_patching_id:
1141       { StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments);
1142         // we should set up register map
1143         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
1144       }
1145       break;
1146 
1147     case handle_exception_nofpu_id:
1148     case handle_exception_id:
1149       { StubFrame f(sasm, "handle_exception", dont_gc_arguments);
1150         oop_maps = generate_handle_exception(id, sasm);
1151       }
1152       break;
1153 
1154     case handle_exception_from_callee_id:
1155       { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
1156         oop_maps = generate_handle_exception(id, sasm);
1157       }
1158       break;
1159 
1160     case throw_index_exception_id:
1161       { StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments);
1162         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
1163       }
1164       break;
1165 
1166     case throw_array_store_exception_id:
1167       { StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments);
1168         // tos + 0: link
1169         //     + 1: return address
1170         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
1171       }
1172       break;
1173 
1174 #if INCLUDE_ALL_GCS
1175 
1176     case g1_pre_barrier_slow_id:
1177       {
1178         StubFrame f(sasm, "g1_pre_barrier", dont_gc_arguments);
1179         // arg0 : previous value of memory
1180 
1181         BarrierSet* bs = Universe::heap()->barrier_set();
1182         if (bs->kind() != BarrierSet::G1SATBCTLogging && bs->kind() != BarrierSet::ShenandoahBarrierSet) {
1183           __ mov(r0, (int)id);
1184           __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1185           __ should_not_reach_here();
1186           break;
1187         }
1188 
1189         const Register pre_val = r0;
1190         const Register thread = rthread;
1191         const Register tmp = rscratch1;
1192 
1193         Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1194                                              PtrQueue::byte_offset_of_active()));
1195 
1196         Address queue_index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1197                                              PtrQueue::byte_offset_of_index()));
1198         Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1199                                         PtrQueue::byte_offset_of_buf()));
1200 
1201         Label done;
1202         Label runtime;
1203 
1204         // Can we store original value in the thread's buffer?
1205         __ ldr(tmp, queue_index);
1206         __ cbz(tmp, runtime);
1207 
1208         __ sub(tmp, tmp, wordSize);
1209         __ str(tmp, queue_index);
1210         __ ldr(rscratch2, buffer);
1211         __ add(tmp, tmp, rscratch2);
1212         f.load_argument(0, rscratch2);
1213         __ str(rscratch2, Address(tmp, 0));
1214         __ b(done);
1215 
1216         __ bind(runtime);
1217         __ push_call_clobbered_registers();
1218         f.load_argument(0, pre_val);
1219         __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
1220         __ pop_call_clobbered_registers();
1221         __ bind(done);
1222       }
1223       break;
1224     case g1_post_barrier_slow_id:
1225       {
1226         StubFrame f(sasm, "g1_post_barrier", dont_gc_arguments);
1227 
1228         // arg0: store_address
1229         Address store_addr(rfp, 2*BytesPerWord);
1230 
1231         BarrierSet* bs = Universe::heap()->barrier_set();
1232         if (bs->kind() == BarrierSet::ShenandoahBarrierSet) {
1233           __ movptr(r0, (int)id);
1234           __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1235           __ should_not_reach_here();
1236           break;
1237         }
1238 
1239         CardTableModRefBS* ct = (CardTableModRefBS*)bs;
1240         assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1241 
1242         Label done;
1243         Label runtime;
1244 
1245         // At this point we know new_value is non-NULL and the new_value crosses regions.
1246         // Must check to see if card is already dirty
1247 
1248         const Register thread = rthread;
1249 
1250         Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
1251                                              PtrQueue::byte_offset_of_index()));
1252         Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
1253                                         PtrQueue::byte_offset_of_buf()));
1254 
1255         const Register card_offset = rscratch2;
1256         // LR is free here, so we can use it to hold the byte_map_base.
1257         const Register byte_map_base = lr;
1258 
1259         assert_different_registers(card_offset, byte_map_base, rscratch1);
1260 
1261         f.load_argument(0, card_offset);
1262         __ lsr(card_offset, card_offset, CardTableModRefBS::card_shift);
1263         __ load_byte_map_base(byte_map_base);
1264         __ ldrb(rscratch1, Address(byte_map_base, card_offset));
1265         __ cmpw(rscratch1, (int)G1SATBCardTableModRefBS::g1_young_card_val());
1266         __ br(Assembler::EQ, done);
1267 
1268         assert((int)CardTableModRefBS::dirty_card_val() == 0, "must be 0");
1269 
1270         __ membar(Assembler::StoreLoad);
1271         __ ldrb(rscratch1, Address(byte_map_base, card_offset));
1272         __ cbzw(rscratch1, done);
1273 
1274         // storing region crossing non-NULL, card is clean.
1275         // dirty card and log.
1276         __ strb(zr, Address(byte_map_base, card_offset));
1277 
1278         // Convert card offset into an address in card_addr
1279         Register card_addr = card_offset;
1280         __ add(card_addr, byte_map_base, card_addr);
1281 
1282         __ ldr(rscratch1, queue_index);
1283         __ cbz(rscratch1, runtime);
1284         __ sub(rscratch1, rscratch1, wordSize);
1285         __ str(rscratch1, queue_index);
1286 
1287         // Reuse LR to hold buffer_addr
1288         const Register buffer_addr = lr;
1289 
1290         __ ldr(buffer_addr, buffer);
1291         __ str(card_addr, Address(buffer_addr, rscratch1));
1292         __ b(done);
1293 
1294         __ bind(runtime);
1295         __ push_call_clobbered_registers();
1296         __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
1297         __ pop_call_clobbered_registers();
1298         __ bind(done);
1299 
1300       }
1301       break;
1302 #endif
1303 
1304     case predicate_failed_trap_id:
1305       {
1306         StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments);
1307 
1308         OopMap* map = save_live_registers(sasm);
1309 
1310         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1311         oop_maps = new OopMapSet();
1312         oop_maps->add_gc_map(call_offset, map);
1313         restore_live_registers(sasm);
1314         __ leave();
1315         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1316         assert(deopt_blob != NULL, "deoptimization blob must have been created");
1317 
1318         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1319       }
1320       break;
1321 
1322 
1323     default:
1324       { StubFrame f(sasm, "unimplemented entry", dont_gc_arguments);
1325         __ mov(r0, (int)id);
1326         __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1327         __ should_not_reach_here();
1328       }
1329       break;
1330     }
1331   }
1332   return oop_maps;
1333 }
1334 
1335 #undef __
1336 
1337 // Simple helper to see if the caller of a runtime stub which
1338 // entered the VM has been deoptimized
1339 
1340 static bool caller_is_deopted() {
1341   JavaThread* thread = JavaThread::current();
1342   RegisterMap reg_map(thread, false);
1343   frame runtime_frame = thread->last_frame();
1344   frame caller_frame = runtime_frame.sender(&reg_map);
1345   assert(caller_frame.is_compiled_frame(), "must be compiled");
1346   return caller_frame.is_deoptimized_frame();
1347 }
1348 
1349 JRT_ENTRY(void, Runtime1::patch_code_aarch64(JavaThread* thread, Runtime1::StubID stub_id ))
1350 {
1351   RegisterMap reg_map(thread, false);
1352 
1353   NOT_PRODUCT(_patch_code_slowcase_cnt++;)
1354   // According to the ARMv8 ARM, "Concurrent modification and
1355   // execution of instructions can lead to the resulting instruction
1356   // performing any behavior that can be achieved by executing any
1357   // sequence of instructions that can be executed from the same
1358   // Exception level, except where the instruction before
1359   // modification and the instruction after modification is a B, BL,
1360   // NOP, BKPT, SVC, HVC, or SMC instruction."
1361   //
1362   // This effectively makes the games we play when patching
1363   // impossible, so when we come across an access that needs
1364   // patching we must deoptimize.
1365 
1366   if (TracePatching) {
1367     tty->print_cr("Deoptimizing because patch is needed");
1368   }
1369 
1370   frame runtime_frame = thread->last_frame();
1371   frame caller_frame = runtime_frame.sender(&reg_map);
1372 
1373   // It's possible the nmethod was invalidated in the last
1374   // safepoint, but if it's still alive then make it not_entrant.
1375   nmethod* nm = CodeCache::find_nmethod(caller_frame.pc());
1376   if (nm != NULL) {
1377     nm->make_not_entrant();
1378   }
1379 
1380   Deoptimization::deoptimize_frame(thread, caller_frame.id());
1381 
1382   // Return to the now deoptimized frame.
1383 }
1384 JRT_END
1385 
1386 int Runtime1::access_field_patching(JavaThread* thread) {
1387 //
1388 // NOTE: we are still in Java
1389 //
1390   Thread* THREAD = thread;
1391   debug_only(NoHandleMark nhm;)
1392   {
1393     // Enter VM mode
1394 
1395     ResetNoHandleMark rnhm;
1396     patch_code_aarch64(thread, access_field_patching_id);
1397   }
1398   // Back in JAVA, use no oops DON'T safepoint
1399 
1400   // Return true if calling code is deoptimized
1401 
1402   return caller_is_deopted();
1403 JRT_END
1404 
1405 
1406 int Runtime1::move_mirror_patching(JavaThread* thread) {
1407 //
1408 // NOTE: we are still in Java
1409 //
1410   Thread* THREAD = thread;
1411   debug_only(NoHandleMark nhm;)
1412   {
1413     // Enter VM mode
1414 
1415     ResetNoHandleMark rnhm;
1416     patch_code_aarch64(thread, load_mirror_patching_id);
1417   }
1418   // Back in JAVA, use no oops DON'T safepoint
1419 
1420   // Return true if calling code is deoptimized
1421 
1422   return caller_is_deopted();
1423 }
1424 
1425 int Runtime1::move_appendix_patching(JavaThread* thread) {
1426 //
1427 // NOTE: we are still in Java
1428 //
1429   Thread* THREAD = thread;
1430   debug_only(NoHandleMark nhm;)
1431   {
1432     // Enter VM mode
1433 
1434     ResetNoHandleMark rnhm;
1435     patch_code_aarch64(thread, load_appendix_patching_id);
1436   }
1437   // Back in JAVA, use no oops DON'T safepoint
1438 
1439   // Return true if calling code is deoptimized
1440 
1441   return caller_is_deopted();
1442 }
1443 
1444 int Runtime1::move_klass_patching(JavaThread* thread) {
1445 //
1446 // NOTE: we are still in Java
1447 //
1448   Thread* THREAD = thread;
1449   debug_only(NoHandleMark nhm;)
1450   {
1451     // Enter VM mode
1452 
1453     ResetNoHandleMark rnhm;
1454     patch_code_aarch64(thread, load_klass_patching_id);
1455   }
1456   // Back in JAVA, use no oops DON'T safepoint
1457 
1458   // Return true if calling code is deoptimized
1459 
1460   return caller_is_deopted();
1461 }
1462 
1463 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }