1 /*
   2  * Copyright (c) 2013, Red Hat Inc.
   3  * Copyright (c) 1999, 2011, Oracle and/or its affiliates.
   4  * All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "c1/c1_CodeStubs.hpp"
  30 #include "c1/c1_Defs.hpp"
  31 #include "c1/c1_MacroAssembler.hpp"
  32 #include "c1/c1_Runtime1.hpp"
  33 #include "compiler/disassembler.hpp"
  34 #include "interpreter/interpreter.hpp"
  35 #include "nativeInst_aarch64.hpp"
  36 #include "oops/compiledICHolder.hpp"
  37 #include "oops/oop.inline.hpp"
  38 #include "prims/jvmtiExport.hpp"
  39 #include "register_aarch64.hpp"
  40 #include "runtime/sharedRuntime.hpp"
  41 #include "runtime/signature.hpp"
  42 #include "runtime/vframe.hpp"
  43 #include "runtime/vframeArray.hpp"
  44 #include "vmreg_aarch64.inline.hpp"
  45 #if INCLUDE_ALL_GCS
  46 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp"
  47 #endif
  48 
  49 
  50 // Implementation of StubAssembler
  51 
  52 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) {
  53   // setup registers
  54   assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different");
  55   assert(oop_result1 != rthread && metadata_result != rthread, "registers must be different");
  56   assert(args_size >= 0, "illegal args_size");
  57   bool align_stack = false;
  58 
  59   mov(c_rarg0, rthread);
  60   set_num_rt_args(0); // Nothing on stack
  61 
  62   Label retaddr;
  63   set_last_Java_frame(sp, rfp, retaddr, rscratch1);
  64 
  65   // do the call
  66   lea(rscratch1, RuntimeAddress(entry));
  67   blr(rscratch1);
  68   bind(retaddr);
  69   int call_offset = offset();
  70   // verify callee-saved register
  71 #ifdef ASSERT
  72   push(r0, sp);
  73   { Label L;
  74     get_thread(r0);
  75     cmp(rthread, r0);
  76     br(Assembler::EQ, L);
  77     stop("StubAssembler::call_RT: rthread not callee saved?");
  78     bind(L);
  79   }
  80   pop(r0, sp);
  81 #endif
  82   reset_last_Java_frame(true);
  83   maybe_isb();
  84 
  85   // check for pending exceptions
  86   { Label L;
  87     // check for pending exceptions (java_thread is set upon return)
  88     ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset())));
  89     cbz(rscratch1, L);
  90     // exception pending => remove activation and forward to exception handler
  91     // make sure that the vm_results are cleared
  92     if (oop_result1->is_valid()) {
  93       str(zr, Address(rthread, JavaThread::vm_result_offset()));
  94     }
  95     if (metadata_result->is_valid()) {
  96       str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
  97     }
  98     if (frame_size() == no_frame_size) {
  99       leave();
 100       far_jump(RuntimeAddress(StubRoutines::forward_exception_entry()));
 101     } else if (_stub_id == Runtime1::forward_exception_id) {
 102       should_not_reach_here();
 103     } else {
 104       far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
 105     }
 106     bind(L);
 107   }
 108   // get oop results if there are any and reset the values in the thread
 109   if (oop_result1->is_valid()) {
 110     get_vm_result(oop_result1, rthread);
 111   }
 112   if (metadata_result->is_valid()) {
 113     get_vm_result_2(metadata_result, rthread);
 114   }
 115   return call_offset;
 116 }
 117 
 118 
 119 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) {
 120   mov(c_rarg1, arg1);
 121   return call_RT(oop_result1, metadata_result, entry, 1);
 122 }
 123 
 124 
 125 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) {
 126   if (c_rarg1 == arg2) {
 127     if (c_rarg2 == arg1) {
 128       mov(rscratch1, arg1);
 129       mov(arg1, arg2);
 130       mov(arg2, rscratch1);
 131     } else {
 132       mov(c_rarg2, arg2);
 133       mov(c_rarg1, arg1);
 134     }
 135   } else {
 136     mov(c_rarg1, arg1);
 137     mov(c_rarg2, arg2);
 138   }
 139   return call_RT(oop_result1, metadata_result, entry, 2);
 140 }
 141 
 142 
 143 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) {
 144   // if there is any conflict use the stack
 145   if (arg1 == c_rarg2 || arg1 == c_rarg3 ||
 146       arg2 == c_rarg1 || arg2 == c_rarg3 ||
 147       arg3 == c_rarg1 || arg3 == c_rarg2) {
 148     stp(arg3, arg2, Address(pre(sp, -2 * wordSize)));
 149     stp(arg1, zr, Address(pre(sp, -2 * wordSize)));
 150     ldp(c_rarg1, zr, Address(post(sp, 2 * wordSize)));
 151     ldp(c_rarg3, c_rarg2, Address(post(sp, 2 * wordSize)));
 152   } else {
 153     mov(c_rarg1, arg1);
 154     mov(c_rarg2, arg2);
 155     mov(c_rarg3, arg3);
 156   }
 157   return call_RT(oop_result1, metadata_result, entry, 3);
 158 }
 159 
 160 // Implementation of StubFrame
 161 
 162 class StubFrame: public StackObj {
 163  private:
 164   StubAssembler* _sasm;
 165 
 166  public:
 167   StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments);
 168   void load_argument(int offset_in_words, Register reg);
 169 
 170   ~StubFrame();
 171 };;
 172 
 173 
 174 #define __ _sasm->
 175 
 176 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) {
 177   _sasm = sasm;
 178   __ set_info(name, must_gc_arguments);
 179   __ enter();
 180 }
 181 
 182 // load parameters that were stored with LIR_Assembler::store_parameter
 183 // Note: offsets for store_parameter and load_argument must match
 184 void StubFrame::load_argument(int offset_in_words, Register reg) {
 185   // rbp, + 0: link
 186   //     + 1: return address
 187   //     + 2: argument with offset 0
 188   //     + 3: argument with offset 1
 189   //     + 4: ...
 190 
 191   __ ldr(reg, Address(rfp, (offset_in_words + 2) * BytesPerWord));
 192 }
 193 
 194 
 195 StubFrame::~StubFrame() {
 196   __ leave();
 197   __ ret(lr);
 198 }
 199 
 200 #undef __
 201 
 202 
 203 // Implementation of Runtime1
 204 
 205 #define __ sasm->
 206 
 207 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2;
 208 
 209 // Stack layout for saving/restoring  all the registers needed during a runtime
 210 // call (this includes deoptimization)
 211 // Note: note that users of this frame may well have arguments to some runtime
 212 // while these values are on the stack. These positions neglect those arguments
 213 // but the code in save_live_registers will take the argument count into
 214 // account.
 215 //
 216 
 217 enum reg_save_layout {
 218   reg_save_frame_size = 32 /* float */ + 32 /* integer */
 219 };
 220 
 221 // Save off registers which might be killed by calls into the runtime.
 222 // Tries to smart of about FP registers.  In particular we separate
 223 // saving and describing the FPU registers for deoptimization since we
 224 // have to save the FPU registers twice if we describe them.  The
 225 // deopt blob is the only thing which needs to describe FPU registers.
 226 // In all other cases it should be sufficient to simply save their
 227 // current value.
 228 
 229 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs];
 230 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs];
 231 static int reg_save_size_in_words;
 232 static int frame_size_in_bytes = -1;
 233 
 234 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) {
 235   int frame_size_in_bytes = reg_save_frame_size * BytesPerWord;
 236   sasm->set_frame_size(frame_size_in_bytes / BytesPerWord);
 237   int frame_size_in_slots = frame_size_in_bytes / sizeof(jint);
 238   OopMap* oop_map = new OopMap(frame_size_in_slots, 0);
 239 
 240   for (int i = 0; i < FrameMap::nof_cpu_regs; i++) {
 241     Register r = as_Register(i);
 242     if (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding()) {
 243       int sp_offset = cpu_reg_save_offsets[i];
 244       oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
 245                                 r->as_VMReg());
 246     }
 247   }
 248 
 249   if (save_fpu_registers) {
 250     for (int i = 0; i < FrameMap::nof_fpu_regs; i++) {
 251       FloatRegister r = as_FloatRegister(i);
 252       {
 253         int sp_offset = fpu_reg_save_offsets[i];
 254         oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset),
 255                                   r->as_VMReg());
 256       }
 257     }
 258   }
 259   return oop_map;
 260 }
 261 
 262 static OopMap* save_live_registers(StubAssembler* sasm,
 263                                    bool save_fpu_registers = true) {
 264   __ block_comment("save_live_registers");
 265 
 266   __ push(RegSet::range(r0, r29), sp);         // integer registers except lr & sp
 267 
 268   if (save_fpu_registers) {
 269     for (int i = 30; i >= 0; i -= 2)
 270       __ stpd(as_FloatRegister(i), as_FloatRegister(i+1),
 271               Address(__ pre(sp, -2 * wordSize)));
 272   } else {
 273     __ add(sp, sp, -32 * wordSize);
 274   }
 275 
 276   return generate_oop_map(sasm, save_fpu_registers);
 277 }
 278 
 279 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) {
 280   if (restore_fpu_registers) {
 281     for (int i = 0; i < 32; i += 2)
 282       __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
 283               Address(__ post(sp, 2 * wordSize)));
 284   } else {
 285     __ add(sp, sp, 32 * wordSize);
 286   }
 287 
 288   __ pop(RegSet::range(r0, r29), sp);
 289 }
 290 
 291 static void restore_live_registers_except_r0(StubAssembler* sasm, bool restore_fpu_registers = true)  {
 292 
 293   if (restore_fpu_registers) {
 294     for (int i = 0; i < 32; i += 2)
 295       __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1),
 296               Address(__ post(sp, 2 * wordSize)));
 297   } else {
 298     __ add(sp, sp, 32 * wordSize);
 299   }
 300 
 301   __ ldp(zr, r1, Address(__ post(sp, 16)));
 302   __ pop(RegSet::range(r2, r29), sp);
 303 }
 304 
 305 
 306 
 307 void Runtime1::initialize_pd() {
 308   int i;
 309   int sp_offset = 0;
 310 
 311   // all float registers are saved explicitly
 312   assert(FrameMap::nof_fpu_regs == 32, "double registers not handled here");
 313   for (i = 0; i < FrameMap::nof_fpu_regs; i++) {
 314     fpu_reg_save_offsets[i] = sp_offset;
 315     sp_offset += 2;   // SP offsets are in halfwords
 316   }
 317 
 318   for (i = 0; i < FrameMap::nof_cpu_regs; i++) {
 319     Register r = as_Register(i);
 320     cpu_reg_save_offsets[i] = sp_offset;
 321     sp_offset += 2;   // SP offsets are in halfwords
 322   }
 323 }
 324 
 325 
 326 // target: the entry point of the method that creates and posts the exception oop
 327 // has_argument: true if the exception needs an argument (passed in rscratch1)
 328 
 329 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) {
 330   // make a frame and preserve the caller's caller-save registers
 331   OopMap* oop_map = save_live_registers(sasm);
 332   int call_offset;
 333   if (!has_argument) {
 334     call_offset = __ call_RT(noreg, noreg, target);
 335   } else {
 336     call_offset = __ call_RT(noreg, noreg, target, rscratch1);
 337   }
 338   OopMapSet* oop_maps = new OopMapSet();
 339   oop_maps->add_gc_map(call_offset, oop_map);
 340 
 341   __ should_not_reach_here();
 342   return oop_maps;
 343 }
 344 
 345 
 346 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) {
 347   __ block_comment("generate_handle_exception");
 348 
 349   // incoming parameters
 350   const Register exception_oop = r0;
 351   const Register exception_pc  = r3;
 352   // other registers used in this stub
 353 
 354   // Save registers, if required.
 355   OopMapSet* oop_maps = new OopMapSet();
 356   OopMap* oop_map = NULL;
 357   switch (id) {
 358   case forward_exception_id:
 359     // We're handling an exception in the context of a compiled frame.
 360     // The registers have been saved in the standard places.  Perform
 361     // an exception lookup in the caller and dispatch to the handler
 362     // if found.  Otherwise unwind and dispatch to the callers
 363     // exception handler.
 364     oop_map = generate_oop_map(sasm, 1 /*thread*/);
 365 
 366     // load and clear pending exception oop into r0
 367     __ ldr(exception_oop, Address(rthread, Thread::pending_exception_offset()));
 368     __ str(zr, Address(rthread, Thread::pending_exception_offset()));
 369 
 370     // load issuing PC (the return address for this stub) into r3
 371     __ ldr(exception_pc, Address(rfp, 1*BytesPerWord));
 372 
 373     // make sure that the vm_results are cleared (may be unnecessary)
 374     __ str(zr, Address(rthread, JavaThread::vm_result_offset()));
 375     __ str(zr, Address(rthread, JavaThread::vm_result_2_offset()));
 376     break;
 377   case handle_exception_nofpu_id:
 378   case handle_exception_id:
 379     // At this point all registers MAY be live.
 380     oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id);
 381     break;
 382   case handle_exception_from_callee_id: {
 383     // At this point all registers except exception oop (r0) and
 384     // exception pc (lr) are dead.
 385     const int frame_size = 2 /*fp, return address*/;
 386     oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0);
 387     sasm->set_frame_size(frame_size);
 388     break;
 389   }
 390   default:
 391     __ should_not_reach_here();
 392     break;
 393   }
 394 
 395   // verify that only r0 and r3 are valid at this time
 396   __ invalidate_registers(false, true, true, false, true, true);
 397   // verify that r0 contains a valid exception
 398   __ verify_not_null_oop(exception_oop);
 399 
 400 #ifdef ASSERT
 401   // check that fields in JavaThread for exception oop and issuing pc are
 402   // empty before writing to them
 403   Label oop_empty;
 404   __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 405   __ cbz(rscratch1, oop_empty);
 406   __ stop("exception oop already set");
 407   __ bind(oop_empty);
 408 
 409   Label pc_empty;
 410   __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 411   __ cbz(rscratch1, pc_empty);
 412   __ stop("exception pc already set");
 413   __ bind(pc_empty);
 414 #endif
 415 
 416   // save exception oop and issuing pc into JavaThread
 417   // (exception handler will load it from here)
 418   __ str(exception_oop, Address(rthread, JavaThread::exception_oop_offset()));
 419   __ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset()));
 420 
 421   // patch throwing pc into return address (has bci & oop map)
 422   __ str(exception_pc, Address(rfp, 1*BytesPerWord));
 423 
 424   // compute the exception handler.
 425   // the exception oop and the throwing pc are read from the fields in JavaThread
 426   int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc));
 427   oop_maps->add_gc_map(call_offset, oop_map);
 428 
 429   // r0: handler address
 430   //      will be the deopt blob if nmethod was deoptimized while we looked up
 431   //      handler regardless of whether handler existed in the nmethod.
 432 
 433   // only r0 is valid at this time, all other registers have been destroyed by the runtime call
 434   __ invalidate_registers(false, true, true, true, true, true);
 435 
 436   // patch the return address, this stub will directly return to the exception handler
 437   __ str(r0, Address(rfp, 1*BytesPerWord));
 438 
 439   switch (id) {
 440   case forward_exception_id:
 441   case handle_exception_nofpu_id:
 442   case handle_exception_id:
 443     // Restore the registers that were saved at the beginning.
 444     restore_live_registers(sasm, id != handle_exception_nofpu_id);
 445     break;
 446   case handle_exception_from_callee_id:
 447     // WIN64_ONLY: No need to add frame::arg_reg_save_area_bytes to SP
 448     // since we do a leave anyway.
 449 
 450     // Pop the return address since we are possibly changing SP (restoring from BP).
 451     __ leave();
 452 
 453     // Restore SP from FP if the exception PC is a method handle call site.
 454     {
 455       Label nope;
 456       __ ldrw(rscratch1, Address(rthread, JavaThread::is_method_handle_return_offset()));
 457       __ cbzw(rscratch1, nope);
 458       __ mov(sp, rfp);
 459       __ bind(nope);
 460     }
 461 
 462     __ ret(lr);  // jump to exception handler
 463     break;
 464   default:  ShouldNotReachHere();
 465   }
 466 
 467   return oop_maps;
 468 }
 469 
 470 
 471 void Runtime1::generate_unwind_exception(StubAssembler *sasm) {
 472   // incoming parameters
 473   const Register exception_oop = r0;
 474   // callee-saved copy of exception_oop during runtime call
 475   const Register exception_oop_callee_saved = r19;
 476   // other registers used in this stub
 477   const Register exception_pc = r3;
 478   const Register handler_addr = r1;
 479 
 480   // verify that only r0, is valid at this time
 481   __ invalidate_registers(false, true, true, true, true, true);
 482 
 483 #ifdef ASSERT
 484   // check that fields in JavaThread for exception oop and issuing pc are empty
 485   Label oop_empty;
 486   __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset()));
 487   __ cbz(rscratch1, oop_empty);
 488   __ stop("exception oop must be empty");
 489   __ bind(oop_empty);
 490 
 491   Label pc_empty;
 492   __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 493   __ cbz(rscratch1, pc_empty);
 494   __ stop("exception pc must be empty");
 495   __ bind(pc_empty);
 496 #endif
 497 
 498   // Save our return address because
 499   // exception_handler_for_return_address will destroy it.  We also
 500   // save exception_oop
 501   __ stp(lr, exception_oop, Address(__ pre(sp, -2 * wordSize)));
 502 
 503   // search the exception handler address of the caller (using the return address)
 504   __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, lr);
 505   // r0: exception handler address of the caller
 506 
 507   // Only R0 is valid at this time; all other registers have been
 508   // destroyed by the call.
 509   __ invalidate_registers(false, true, true, true, false, true);
 510 
 511   // move result of call into correct register
 512   __ mov(handler_addr, r0);
 513 
 514   // get throwing pc (= return address).
 515   // lr has been destroyed by the call
 516   __ ldp(lr, exception_oop, Address(__ post(sp, 2 * wordSize)));
 517   __ mov(r3, lr);
 518 
 519   __ verify_not_null_oop(exception_oop);
 520 
 521   {
 522     Label foo;
 523     __ ldrw(rscratch1, Address(rthread, JavaThread::is_method_handle_return_offset()));
 524     __ cbzw(rscratch1, foo);
 525     __ mov(sp, rfp);
 526     __ bind(foo);
 527   }
 528 
 529   // continue at exception handler (return address removed)
 530   // note: do *not* remove arguments when unwinding the
 531   //       activation since the caller assumes having
 532   //       all arguments on the stack when entering the
 533   //       runtime to determine the exception handler
 534   //       (GC happens at call site with arguments!)
 535   // r0: exception oop
 536   // r3: throwing pc
 537   // r1: exception handler
 538   __ br(handler_addr);
 539 }
 540 
 541 
 542 
 543 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) {
 544   // use the maximum number of runtime-arguments here because it is difficult to
 545   // distinguish each RT-Call.
 546   // Note: This number affects also the RT-Call in generate_handle_exception because
 547   //       the oop-map is shared for all calls.
 548   DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
 549   assert(deopt_blob != NULL, "deoptimization blob must have been created");
 550 
 551   OopMap* oop_map = save_live_registers(sasm);
 552 
 553   __ mov(c_rarg0, rthread);
 554   Label retaddr;
 555   __ set_last_Java_frame(sp, rfp, retaddr, rscratch1);
 556   // do the call
 557   __ lea(rscratch1, RuntimeAddress(target));
 558   __ blr(rscratch1);
 559   __ bind(retaddr);
 560   OopMapSet* oop_maps = new OopMapSet();
 561   oop_maps->add_gc_map(__ offset(), oop_map);
 562   // verify callee-saved register
 563 #ifdef ASSERT
 564   { Label L;
 565     __ get_thread(rscratch1);
 566     __ cmp(rthread, rscratch1);
 567     __ br(Assembler::EQ, L);
 568     __ stop("StubAssembler::call_RT: rthread not callee saved?");
 569     __ bind(L);
 570   }
 571 #endif
 572   __ reset_last_Java_frame(true);
 573   __ maybe_isb();
 574 
 575   // check for pending exceptions
 576   { Label L;
 577     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 578     __ cbz(rscratch1, L);
 579     // exception pending => remove activation and forward to exception handler
 580 
 581     { Label L1;
 582       __ cbnz(r0, L1);                                  // have we deoptimized?
 583       __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id)));
 584       __ bind(L1);
 585     }
 586 
 587     // the deopt blob expects exceptions in the special fields of
 588     // JavaThread, so copy and clear pending exception.
 589 
 590     // load and clear pending exception
 591     __ ldr(r0, Address(rthread, Thread::pending_exception_offset()));
 592     __ str(zr, Address(rthread, Thread::pending_exception_offset()));
 593 
 594     // check that there is really a valid exception
 595     __ verify_not_null_oop(r0);
 596 
 597     // load throwing pc: this is the return address of the stub
 598     __ mov(r3, lr);
 599 
 600 #ifdef ASSERT
 601     // check that fields in JavaThread for exception oop and issuing pc are empty
 602     Label oop_empty;
 603     __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset()));
 604     __ cbz(rscratch1, oop_empty);
 605     __ stop("exception oop must be empty");
 606     __ bind(oop_empty);
 607 
 608     Label pc_empty;
 609     __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset()));
 610     __ cbz(rscratch1, pc_empty);
 611     __ stop("exception pc must be empty");
 612     __ bind(pc_empty);
 613 #endif
 614 
 615     // store exception oop and throwing pc to JavaThread
 616     __ str(r0, Address(rthread, JavaThread::exception_oop_offset()));
 617     __ str(r3, Address(rthread, JavaThread::exception_pc_offset()));
 618 
 619     restore_live_registers(sasm);
 620 
 621     __ leave();
 622 
 623     // Forward the exception directly to deopt blob. We can blow no
 624     // registers and must leave throwing pc on the stack.  A patch may
 625     // have values live in registers so the entry point with the
 626     // exception in tls.
 627     __ far_jump(RuntimeAddress(deopt_blob->unpack_with_exception_in_tls()));
 628 
 629     __ bind(L);
 630   }
 631 
 632 
 633   // Runtime will return true if the nmethod has been deoptimized during
 634   // the patching process. In that case we must do a deopt reexecute instead.
 635 
 636   Label reexecuteEntry, cont;
 637 
 638   __ cbz(r0, cont);                                 // have we deoptimized?
 639 
 640   // Will reexecute. Proper return address is already on the stack we just restore
 641   // registers, pop all of our frame but the return address and jump to the deopt blob
 642   restore_live_registers(sasm);
 643   __ leave();
 644   __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
 645 
 646   __ bind(cont);
 647   restore_live_registers(sasm);
 648   __ leave();
 649   __ ret(lr);
 650 
 651   return oop_maps;
 652 }
 653 
 654 
 655 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) {
 656 
 657   const Register exception_oop = r0;
 658   const Register exception_pc  = r3;
 659 
 660   // for better readability
 661   const bool must_gc_arguments = true;
 662   const bool dont_gc_arguments = false;
 663 
 664   // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu
 665   bool save_fpu_registers = true;
 666 
 667   // stub code & info for the different stubs
 668   OopMapSet* oop_maps = NULL;
 669   OopMap* oop_map = NULL;
 670   switch (id) {
 671     {
 672     case forward_exception_id:
 673       {
 674         oop_maps = generate_handle_exception(id, sasm);
 675         __ leave();
 676         __ ret(lr);
 677       }
 678       break;
 679 
 680     case throw_div0_exception_id:
 681       { StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments);
 682         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false);
 683       }
 684       break;
 685 
 686     case throw_null_pointer_exception_id:
 687       { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments);
 688         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false);
 689       }
 690       break;
 691 
 692     case new_instance_id:
 693     case fast_new_instance_id:
 694     case fast_new_instance_init_check_id:
 695       {
 696         Register klass = r3; // Incoming
 697         Register obj   = r0; // Result
 698 
 699         if (id == new_instance_id) {
 700           __ set_info("new_instance", dont_gc_arguments);
 701         } else if (id == fast_new_instance_id) {
 702           __ set_info("fast new_instance", dont_gc_arguments);
 703         } else {
 704           assert(id == fast_new_instance_init_check_id, "bad StubID");
 705           __ set_info("fast new_instance init check", dont_gc_arguments);
 706         }
 707 
 708         if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) &&
 709             UseTLAB && FastTLABRefill) {
 710           Label slow_path;
 711           Register obj_size = r2;
 712           Register t1       = r19;
 713           Register t2       = r4;
 714           assert_different_registers(klass, obj, obj_size, t1, t2);
 715 
 716           __ stp(r5, r19, Address(__ pre(sp, -2 * wordSize)));
 717 
 718           if (id == fast_new_instance_init_check_id) {
 719             // make sure the klass is initialized
 720             __ ldrb(rscratch1, Address(klass, InstanceKlass::init_state_offset()));
 721             __ cmpw(rscratch1, InstanceKlass::fully_initialized);
 722             __ br(Assembler::NE, slow_path);
 723           }
 724 
 725 #ifdef ASSERT
 726           // assert object can be fast path allocated
 727           {
 728             Label ok, not_ok;
 729             __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 730             __ cmp(obj_size, 0u);
 731             __ br(Assembler::LE, not_ok);  // make sure it's an instance (LH > 0)
 732             __ tstw(obj_size, Klass::_lh_instance_slow_path_bit);
 733             __ br(Assembler::EQ, ok);
 734             __ bind(not_ok);
 735             __ stop("assert(can be fast path allocated)");
 736             __ should_not_reach_here();
 737             __ bind(ok);
 738           }
 739 #endif // ASSERT
 740 
 741           // if we got here then the TLAB allocation failed, so try
 742           // refilling the TLAB or allocating directly from eden.
 743           Label retry_tlab, try_eden;
 744           __ tlab_refill(retry_tlab, try_eden, slow_path); // does not destroy r3 (klass), returns r5
 745 
 746           __ bind(retry_tlab);
 747 
 748           // get the instance size (size is postive so movl is fine for 64bit)
 749           __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 750 
 751           __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path);
 752 
 753           __ initialize_object(obj, klass, obj_size, 0, t1, t2);
 754           __ verify_oop(obj);
 755           __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
 756           __ ret(lr);
 757 
 758           __ bind(try_eden);
 759           // get the instance size (size is postive so movl is fine for 64bit)
 760           __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset()));
 761 
 762           __ eden_allocate(obj, obj_size, 0, t1, slow_path);
 763           __ incr_allocated_bytes(rthread, obj_size, 0, rscratch1);
 764 
 765           __ initialize_object(obj, klass, obj_size, 0, t1, t2);
 766           __ verify_oop(obj);
 767           __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
 768           __ ret(lr);
 769 
 770           __ bind(slow_path);
 771           __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize)));
 772         }
 773 
 774         __ enter();
 775         OopMap* map = save_live_registers(sasm);
 776         int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass);
 777         oop_maps = new OopMapSet();
 778         oop_maps->add_gc_map(call_offset, map);
 779         restore_live_registers_except_r0(sasm);
 780         __ verify_oop(obj);
 781         __ leave();
 782         __ ret(lr);
 783 
 784         // r0,: new instance
 785       }
 786 
 787       break;
 788 
 789     case counter_overflow_id:
 790       {
 791         Register bci = r0, method = r1;
 792         __ enter();
 793         OopMap* map = save_live_registers(sasm);
 794         // Retrieve bci
 795         __ ldrw(bci, Address(rfp, 2*BytesPerWord));
 796         // And a pointer to the Method*
 797         __ ldr(method, Address(rfp, 3*BytesPerWord));
 798         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method);
 799         oop_maps = new OopMapSet();
 800         oop_maps->add_gc_map(call_offset, map);
 801         restore_live_registers(sasm);
 802         __ leave();
 803         __ ret(lr);
 804       }
 805       break;
 806 
 807     case new_type_array_id:
 808     case new_object_array_id:
 809       {
 810         Register length   = r19; // Incoming
 811         Register klass    = r3; // Incoming
 812         Register obj      = r0; // Result
 813 
 814         if (id == new_type_array_id) {
 815           __ set_info("new_type_array", dont_gc_arguments);
 816         } else {
 817           __ set_info("new_object_array", dont_gc_arguments);
 818         }
 819 
 820 #ifdef ASSERT
 821         // assert object type is really an array of the proper kind
 822         {
 823           Label ok;
 824           Register t0 = obj;
 825           __ ldrw(t0, Address(klass, Klass::layout_helper_offset()));
 826           __ asrw(t0, t0, Klass::_lh_array_tag_shift);
 827           int tag = ((id == new_type_array_id)
 828                      ? Klass::_lh_array_tag_type_value
 829                      : Klass::_lh_array_tag_obj_value);
 830           __ mov(rscratch1, tag);
 831           __ cmpw(t0, rscratch1);
 832           __ br(Assembler::EQ, ok);
 833           __ stop("assert(is an array klass)");
 834           __ should_not_reach_here();
 835           __ bind(ok);
 836         }
 837 #endif // ASSERT
 838 
 839         if (UseTLAB && FastTLABRefill) {
 840           Register arr_size = r4;
 841           Register t1       = r2;
 842           Register t2       = r5;
 843           Label slow_path;
 844           assert_different_registers(length, klass, obj, arr_size, t1, t2);
 845 
 846           // check that array length is small enough for fast path.
 847           __ mov(rscratch1, C1_MacroAssembler::max_array_allocation_length);
 848           __ cmpw(length, rscratch1);
 849           __ br(Assembler::HI, slow_path);
 850 
 851           // if we got here then the TLAB allocation failed, so try
 852           // refilling the TLAB or allocating directly from eden.
 853           Label retry_tlab, try_eden;
 854           const Register thread =
 855             __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves r19 & r3, returns rthread
 856 
 857           __ bind(retry_tlab);
 858 
 859           // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
 860           // since size is positive ldrw does right thing on 64bit
 861           __ ldrw(t1, Address(klass, Klass::layout_helper_offset()));
 862           __ lslvw(arr_size, length, t1);
 863           __ ubfx(t1, t1, Klass::_lh_header_size_shift,
 864                   exact_log2(Klass::_lh_header_size_mask + 1));
 865           __ add(arr_size, arr_size, t1);
 866           __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
 867           __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
 868 
 869           __ tlab_allocate(obj, arr_size, 0, t1, t2, slow_path);  // preserves arr_size
 870 
 871           __ initialize_header(obj, klass, length, t1, t2);
 872           __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte)));
 873           assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
 874           assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
 875           __ andr(t1, t1, Klass::_lh_header_size_mask);
 876           __ sub(arr_size, arr_size, t1);  // body length
 877           __ add(t1, t1, obj);       // body start
 878           __ initialize_body(t1, arr_size, 0, t2);
 879           __ membar(Assembler::StoreStore);
 880           __ verify_oop(obj);
 881 
 882           __ ret(lr);
 883 
 884           __ bind(try_eden);
 885           // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F))
 886           // since size is positive ldrw does right thing on 64bit
 887           __ ldrw(t1, Address(klass, Klass::layout_helper_offset()));
 888           // since size is postive movw does right thing on 64bit
 889           __ movw(arr_size, length);
 890           __ lslvw(arr_size, length, t1);
 891           __ ubfx(t1, t1, Klass::_lh_header_size_shift,
 892                   exact_log2(Klass::_lh_header_size_mask + 1));
 893           __ add(arr_size, arr_size, t1);
 894           __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up
 895           __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask);
 896 
 897           __ eden_allocate(obj, arr_size, 0, t1, slow_path);  // preserves arr_size
 898           __ incr_allocated_bytes(thread, arr_size, 0, rscratch1);
 899 
 900           __ initialize_header(obj, klass, length, t1, t2);
 901           __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte)));
 902           assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise");
 903           assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise");
 904           __ andr(t1, t1, Klass::_lh_header_size_mask);
 905           __ sub(arr_size, arr_size, t1);  // body length
 906           __ add(t1, t1, obj);       // body start
 907           __ initialize_body(t1, arr_size, 0, t2);
 908           __ membar(Assembler::StoreStore);
 909           __ verify_oop(obj);
 910 
 911           __ ret(lr);
 912 
 913           __ bind(slow_path);
 914         }
 915 
 916         __ enter();
 917         OopMap* map = save_live_registers(sasm);
 918         int call_offset;
 919         if (id == new_type_array_id) {
 920           call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length);
 921         } else {
 922           call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length);
 923         }
 924 
 925         oop_maps = new OopMapSet();
 926         oop_maps->add_gc_map(call_offset, map);
 927         restore_live_registers_except_r0(sasm);
 928 
 929         __ verify_oop(obj);
 930         __ leave();
 931         __ ret(lr);
 932 
 933         // r0: new array
 934       }
 935       break;
 936 
 937     case new_multi_array_id:
 938       { StubFrame f(sasm, "new_multi_array", dont_gc_arguments);
 939         // r0,: klass
 940         // r19,: rank
 941         // r2: address of 1st dimension
 942         OopMap* map = save_live_registers(sasm);
 943         __ mov(c_rarg1, r0);
 944         __ mov(c_rarg3, r2);
 945         __ mov(c_rarg2, r19);
 946         int call_offset = __ call_RT(r0, noreg, CAST_FROM_FN_PTR(address, new_multi_array), r1, r2, r3);
 947 
 948         oop_maps = new OopMapSet();
 949         oop_maps->add_gc_map(call_offset, map);
 950         restore_live_registers_except_r0(sasm);
 951 
 952         // r0,: new multi array
 953         __ verify_oop(r0);
 954       }
 955       break;
 956 
 957     case register_finalizer_id:
 958       {
 959         __ set_info("register_finalizer", dont_gc_arguments);
 960 
 961         // This is called via call_runtime so the arguments
 962         // will be place in C abi locations
 963 
 964         __ verify_oop(c_rarg0);
 965 
 966         // load the klass and check the has finalizer flag
 967         Label register_finalizer;
 968         Register t = r5;
 969         __ load_klass(t, r0);
 970         __ ldrw(t, Address(t, Klass::access_flags_offset()));
 971         __ tst(t, JVM_ACC_HAS_FINALIZER);
 972         __ br(Assembler::NE, register_finalizer);
 973         __ ret(lr);
 974 
 975         __ bind(register_finalizer);
 976         __ enter();
 977         OopMap* oop_map = save_live_registers(sasm);
 978         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), r0);
 979         oop_maps = new OopMapSet();
 980         oop_maps->add_gc_map(call_offset, oop_map);
 981 
 982         // Now restore all the live registers
 983         restore_live_registers(sasm);
 984 
 985         __ leave();
 986         __ ret(lr);
 987       }
 988       break;
 989 
 990     case throw_class_cast_exception_id:
 991       { StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments);
 992         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true);
 993       }
 994       break;
 995 
 996     case throw_incompatible_class_change_error_id:
 997       { StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments);
 998         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false);
 999       }
1000       break;
1001 
1002     case slow_subtype_check_id:
1003       {
1004         // Typical calling sequence:
1005         // __ push(klass_RInfo);  // object klass or other subclass
1006         // __ push(sup_k_RInfo);  // array element klass or other superclass
1007         // __ bl(slow_subtype_check);
1008         // Note that the subclass is pushed first, and is therefore deepest.
1009         enum layout {
1010           r0_off, r0_off_hi,
1011           r2_off, r2_off_hi,
1012           r4_off, r4_off_hi,
1013           r5_off, r5_off_hi,
1014           sup_k_off, sup_k_off_hi,
1015           klass_off, klass_off_hi,
1016           framesize,
1017           result_off = sup_k_off
1018         };
1019 
1020         __ set_info("slow_subtype_check", dont_gc_arguments);
1021         __ push(RegSet::of(r0, r2, r4, r5), sp);
1022 
1023         // This is called by pushing args and not with C abi
1024         // __ ldr(r4, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // subclass
1025         // __ ldr(r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // superclass
1026 
1027         __ ldp(r4, r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size));
1028 
1029         Label miss;
1030         __ check_klass_subtype_slow_path(r4, r0, r2, r5, NULL, &miss);
1031 
1032         // fallthrough on success:
1033         __ mov(rscratch1, 1);
1034         __ str(rscratch1, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
1035         __ pop(RegSet::of(r0, r2, r4, r5), sp);
1036         __ ret(lr);
1037 
1038         __ bind(miss);
1039         __ str(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result
1040         __ pop(RegSet::of(r0, r2, r4, r5), sp);
1041         __ ret(lr);
1042       }
1043       break;
1044 
1045     case monitorenter_nofpu_id:
1046       save_fpu_registers = false;
1047       // fall through
1048     case monitorenter_id:
1049       {
1050         StubFrame f(sasm, "monitorenter", dont_gc_arguments);
1051         OopMap* map = save_live_registers(sasm, save_fpu_registers);
1052 
1053         // Called with store_parameter and not C abi
1054 
1055         f.load_argument(1, r0); // r0,: object
1056         f.load_argument(0, r1); // r1,: lock address
1057 
1058         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), r0, r1);
1059 
1060         oop_maps = new OopMapSet();
1061         oop_maps->add_gc_map(call_offset, map);
1062         restore_live_registers(sasm, save_fpu_registers);
1063       }
1064       break;
1065 
1066     case monitorexit_nofpu_id:
1067       save_fpu_registers = false;
1068       // fall through
1069     case monitorexit_id:
1070       {
1071         StubFrame f(sasm, "monitorexit", dont_gc_arguments);
1072         OopMap* map = save_live_registers(sasm, save_fpu_registers);
1073 
1074         // Called with store_parameter and not C abi
1075 
1076         f.load_argument(0, r0); // r0,: lock address
1077 
1078         // note: really a leaf routine but must setup last java sp
1079         //       => use call_RT for now (speed can be improved by
1080         //       doing last java sp setup manually)
1081         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), r0);
1082 
1083         oop_maps = new OopMapSet();
1084         oop_maps->add_gc_map(call_offset, map);
1085         restore_live_registers(sasm, save_fpu_registers);
1086       }
1087       break;
1088 
1089     case deoptimize_id:
1090       {
1091         StubFrame f(sasm, "deoptimize", dont_gc_arguments);
1092         OopMap* oop_map = save_live_registers(sasm);
1093         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize));
1094         oop_maps = new OopMapSet();
1095         oop_maps->add_gc_map(call_offset, oop_map);
1096         restore_live_registers(sasm);
1097         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1098         assert(deopt_blob != NULL, "deoptimization blob must have been created");
1099         __ leave();
1100         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1101       }
1102       break;
1103 
1104     case throw_range_check_failed_id:
1105       { StubFrame f(sasm, "range_check_failed", dont_gc_arguments);
1106         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true);
1107       }
1108       break;
1109 
1110     case unwind_exception_id:
1111       { __ set_info("unwind_exception", dont_gc_arguments);
1112         // note: no stubframe since we are about to leave the current
1113         //       activation and we are calling a leaf VM function only.
1114         generate_unwind_exception(sasm);
1115       }
1116       break;
1117 
1118     case access_field_patching_id:
1119       { StubFrame f(sasm, "access_field_patching", dont_gc_arguments);
1120         // we should set up register map
1121         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching));
1122       }
1123       break;
1124 
1125     case load_klass_patching_id:
1126       { StubFrame f(sasm, "load_klass_patching", dont_gc_arguments);
1127         // we should set up register map
1128         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching));
1129       }
1130       break;
1131 
1132     case load_mirror_patching_id:
1133       { StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments);
1134         // we should set up register map
1135         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching));
1136       }
1137       break;
1138 
1139     case load_appendix_patching_id:
1140       { StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments);
1141         // we should set up register map
1142         oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching));
1143       }
1144       break;
1145 
1146     case handle_exception_nofpu_id:
1147     case handle_exception_id:
1148       { StubFrame f(sasm, "handle_exception", dont_gc_arguments);
1149         oop_maps = generate_handle_exception(id, sasm);
1150       }
1151       break;
1152 
1153     case handle_exception_from_callee_id:
1154       { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments);
1155         oop_maps = generate_handle_exception(id, sasm);
1156       }
1157       break;
1158 
1159     case throw_index_exception_id:
1160       { StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments);
1161         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true);
1162       }
1163       break;
1164 
1165     case throw_array_store_exception_id:
1166       { StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments);
1167         // tos + 0: link
1168         //     + 1: return address
1169         oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true);
1170       }
1171       break;
1172 
1173 #if INCLUDE_ALL_GCS
1174 
1175     case g1_pre_barrier_slow_id:
1176       {
1177         StubFrame f(sasm, "g1_pre_barrier", dont_gc_arguments);
1178         // arg0 : previous value of memory
1179 
1180         BarrierSet* bs = Universe::heap()->barrier_set();
1181         if (bs->kind() != BarrierSet::G1SATBCTLogging) {
1182           __ mov(r0, (int)id);
1183           __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1184           __ should_not_reach_here();
1185           break;
1186         }
1187 
1188         const Register pre_val = r0;
1189         const Register thread = rthread;
1190         const Register tmp = rscratch1;
1191 
1192         Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1193                                              PtrQueue::byte_offset_of_active()));
1194 
1195         Address queue_index(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1196                                              PtrQueue::byte_offset_of_index()));
1197         Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() +
1198                                         PtrQueue::byte_offset_of_buf()));
1199 
1200         Label done;
1201         Label runtime;
1202 
1203         // Can we store original value in the thread's buffer?
1204         __ ldr(tmp, queue_index);
1205         __ cbz(tmp, runtime);
1206 
1207         __ sub(tmp, tmp, wordSize);
1208         __ str(tmp, queue_index);
1209         __ ldr(rscratch2, buffer);
1210         __ add(tmp, tmp, rscratch2);
1211         f.load_argument(0, rscratch2);
1212         __ str(rscratch2, Address(tmp, 0));
1213         __ b(done);
1214 
1215         __ bind(runtime);
1216         __ push_call_clobbered_registers();
1217         f.load_argument(0, pre_val);
1218         __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread);
1219         __ pop_call_clobbered_registers();
1220         __ bind(done);
1221       }
1222       break;
1223     case g1_post_barrier_slow_id:
1224       {
1225         StubFrame f(sasm, "g1_post_barrier", dont_gc_arguments);
1226 
1227         // arg0: store_address
1228         Address store_addr(rfp, 2*BytesPerWord);
1229 
1230         BarrierSet* bs = Universe::heap()->barrier_set();
1231         CardTableModRefBS* ct = (CardTableModRefBS*)bs;
1232         assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code");
1233 
1234         Label done;
1235         Label runtime;
1236 
1237         // At this point we know new_value is non-NULL and the new_value crosses regions.
1238         // Must check to see if card is already dirty
1239 
1240         const Register thread = rthread;
1241 
1242         Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
1243                                              PtrQueue::byte_offset_of_index()));
1244         Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() +
1245                                         PtrQueue::byte_offset_of_buf()));
1246 
1247         const Register card_offset = rscratch2;
1248         // LR is free here, so we can use it to hold the byte_map_base.
1249         const Register byte_map_base = lr;
1250 
1251         assert_different_registers(card_offset, byte_map_base, rscratch1);
1252 
1253         f.load_argument(0, card_offset);
1254         __ lsr(card_offset, card_offset, CardTableModRefBS::card_shift);
1255         __ load_byte_map_base(byte_map_base);
1256         __ ldrb(rscratch1, Address(byte_map_base, card_offset));
1257         __ cmpw(rscratch1, (int)G1SATBCardTableModRefBS::g1_young_card_val());
1258         __ br(Assembler::EQ, done);
1259 
1260         assert((int)CardTableModRefBS::dirty_card_val() == 0, "must be 0");
1261 
1262         __ membar(Assembler::StoreLoad);
1263         __ ldrb(rscratch1, Address(byte_map_base, card_offset));
1264         __ cbzw(rscratch1, done);
1265 
1266         // storing region crossing non-NULL, card is clean.
1267         // dirty card and log.
1268         __ strb(zr, Address(byte_map_base, card_offset));
1269 
1270         // Convert card offset into an address in card_addr
1271         Register card_addr = card_offset;
1272         __ add(card_addr, byte_map_base, card_addr);
1273 
1274         __ ldr(rscratch1, queue_index);
1275         __ cbz(rscratch1, runtime);
1276         __ sub(rscratch1, rscratch1, wordSize);
1277         __ str(rscratch1, queue_index);
1278 
1279         // Reuse LR to hold buffer_addr
1280         const Register buffer_addr = lr;
1281 
1282         __ ldr(buffer_addr, buffer);
1283         __ str(card_addr, Address(buffer_addr, rscratch1));
1284         __ b(done);
1285 
1286         __ bind(runtime);
1287         __ push_call_clobbered_registers();
1288         __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread);
1289         __ pop_call_clobbered_registers();
1290         __ bind(done);
1291 
1292       }
1293       break;
1294 #endif
1295 
1296     case predicate_failed_trap_id:
1297       {
1298         StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments);
1299 
1300         OopMap* map = save_live_registers(sasm);
1301 
1302         int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap));
1303         oop_maps = new OopMapSet();
1304         oop_maps->add_gc_map(call_offset, map);
1305         restore_live_registers(sasm);
1306         __ leave();
1307         DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob();
1308         assert(deopt_blob != NULL, "deoptimization blob must have been created");
1309 
1310         __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution()));
1311       }
1312       break;
1313 
1314     case dtrace_object_alloc_id:
1315       { // c_rarg0: object
1316         StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments);
1317         save_live_registers(sasm);
1318 
1319         __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), c_rarg0);
1320 
1321         restore_live_registers(sasm);
1322       }
1323       break;
1324 
1325     default:
1326       { StubFrame f(sasm, "unimplemented entry", dont_gc_arguments);
1327         __ mov(r0, (int)id);
1328         __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0);
1329         __ should_not_reach_here();
1330       }
1331       break;
1332     }
1333   }
1334   return oop_maps;
1335 }
1336 
1337 #undef __
1338 
1339 // Simple helper to see if the caller of a runtime stub which
1340 // entered the VM has been deoptimized
1341 
1342 static bool caller_is_deopted() {
1343   JavaThread* thread = JavaThread::current();
1344   RegisterMap reg_map(thread, false);
1345   frame runtime_frame = thread->last_frame();
1346   frame caller_frame = runtime_frame.sender(&reg_map);
1347   assert(caller_frame.is_compiled_frame(), "must be compiled");
1348   return caller_frame.is_deoptimized_frame();
1349 }
1350 
1351 JRT_ENTRY(void, Runtime1::patch_code_aarch64(JavaThread* thread, Runtime1::StubID stub_id ))
1352 {
1353   RegisterMap reg_map(thread, false);
1354 
1355   NOT_PRODUCT(_patch_code_slowcase_cnt++;)
1356   // According to the ARMv8 ARM, "Concurrent modification and
1357   // execution of instructions can lead to the resulting instruction
1358   // performing any behavior that can be achieved by executing any
1359   // sequence of instructions that can be executed from the same
1360   // Exception level, except where the instruction before
1361   // modification and the instruction after modification is a B, BL,
1362   // NOP, BKPT, SVC, HVC, or SMC instruction."
1363   //
1364   // This effectively makes the games we play when patching
1365   // impossible, so when we come across an access that needs
1366   // patching we must deoptimize.
1367 
1368   if (TracePatching) {
1369     tty->print_cr("Deoptimizing because patch is needed");
1370   }
1371 
1372   frame runtime_frame = thread->last_frame();
1373   frame caller_frame = runtime_frame.sender(&reg_map);
1374 
1375   // It's possible the nmethod was invalidated in the last
1376   // safepoint, but if it's still alive then make it not_entrant.
1377   nmethod* nm = CodeCache::find_nmethod(caller_frame.pc());
1378   if (nm != NULL) {
1379     nm->make_not_entrant();
1380   }
1381 
1382   Deoptimization::deoptimize_frame(thread, caller_frame.id());
1383 
1384   // Return to the now deoptimized frame.
1385 }
1386 JRT_END
1387 
1388 int Runtime1::access_field_patching(JavaThread* thread) {
1389 //
1390 // NOTE: we are still in Java
1391 //
1392   Thread* THREAD = thread;
1393   debug_only(NoHandleMark nhm;)
1394   {
1395     // Enter VM mode
1396 
1397     ResetNoHandleMark rnhm;
1398     patch_code_aarch64(thread, access_field_patching_id);
1399   }
1400   // Back in JAVA, use no oops DON'T safepoint
1401 
1402   // Return true if calling code is deoptimized
1403 
1404   return caller_is_deopted();
1405 JRT_END
1406 
1407 
1408 int Runtime1::move_mirror_patching(JavaThread* thread) {
1409 //
1410 // NOTE: we are still in Java
1411 //
1412   Thread* THREAD = thread;
1413   debug_only(NoHandleMark nhm;)
1414   {
1415     // Enter VM mode
1416 
1417     ResetNoHandleMark rnhm;
1418     patch_code_aarch64(thread, load_mirror_patching_id);
1419   }
1420   // Back in JAVA, use no oops DON'T safepoint
1421 
1422   // Return true if calling code is deoptimized
1423 
1424   return caller_is_deopted();
1425 }
1426 
1427 int Runtime1::move_appendix_patching(JavaThread* thread) {
1428 //
1429 // NOTE: we are still in Java
1430 //
1431   Thread* THREAD = thread;
1432   debug_only(NoHandleMark nhm;)
1433   {
1434     // Enter VM mode
1435 
1436     ResetNoHandleMark rnhm;
1437     patch_code_aarch64(thread, load_appendix_patching_id);
1438   }
1439   // Back in JAVA, use no oops DON'T safepoint
1440 
1441   // Return true if calling code is deoptimized
1442 
1443   return caller_is_deopted();
1444 }
1445 
1446 int Runtime1::move_klass_patching(JavaThread* thread) {
1447 //
1448 // NOTE: we are still in Java
1449 //
1450   Thread* THREAD = thread;
1451   debug_only(NoHandleMark nhm;)
1452   {
1453     // Enter VM mode
1454 
1455     ResetNoHandleMark rnhm;
1456     patch_code_aarch64(thread, load_klass_patching_id);
1457   }
1458   // Back in JAVA, use no oops DON'T safepoint
1459 
1460   // Return true if calling code is deoptimized
1461 
1462   return caller_is_deopted();
1463 }
1464 
1465 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }