1 /* 2 * Copyright (c) 2013, Red Hat Inc. 3 * Copyright (c) 1999, 2011, Oracle and/or its affiliates. 4 * All rights reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 #include "asm/assembler.hpp" 29 #include "c1/c1_CodeStubs.hpp" 30 #include "c1/c1_Defs.hpp" 31 #include "c1/c1_MacroAssembler.hpp" 32 #include "c1/c1_Runtime1.hpp" 33 #include "compiler/disassembler.hpp" 34 #include "interpreter/interpreter.hpp" 35 #include "nativeInst_aarch64.hpp" 36 #include "oops/compiledICHolder.hpp" 37 #include "oops/oop.inline.hpp" 38 #include "prims/jvmtiExport.hpp" 39 #include "register_aarch64.hpp" 40 #include "runtime/sharedRuntime.hpp" 41 #include "runtime/signature.hpp" 42 #include "runtime/vframe.hpp" 43 #include "runtime/vframeArray.hpp" 44 #include "vmreg_aarch64.inline.hpp" 45 #if INCLUDE_ALL_GCS 46 #include "gc_implementation/g1/g1SATBCardTableModRefBS.hpp" 47 #endif 48 49 50 // Implementation of StubAssembler 51 52 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, int args_size) { 53 // setup registers 54 assert(!(oop_result1->is_valid() || metadata_result->is_valid()) || oop_result1 != metadata_result, "registers must be different"); 55 assert(oop_result1 != rthread && metadata_result != rthread, "registers must be different"); 56 assert(args_size >= 0, "illegal args_size"); 57 bool align_stack = false; 58 59 mov(c_rarg0, rthread); 60 set_num_rt_args(0); // Nothing on stack 61 62 Label retaddr; 63 set_last_Java_frame(sp, rfp, retaddr, rscratch1); 64 65 // do the call 66 lea(rscratch1, RuntimeAddress(entry)); 67 blr(rscratch1); 68 bind(retaddr); 69 int call_offset = offset(); 70 // verify callee-saved register 71 #ifdef ASSERT 72 push(r0, sp); 73 { Label L; 74 get_thread(r0); 75 cmp(rthread, r0); 76 br(Assembler::EQ, L); 77 stop("StubAssembler::call_RT: rthread not callee saved?"); 78 bind(L); 79 } 80 pop(r0, sp); 81 #endif 82 reset_last_Java_frame(true); 83 maybe_isb(); 84 85 // check for pending exceptions 86 { Label L; 87 // check for pending exceptions (java_thread is set upon return) 88 ldr(rscratch1, Address(rthread, in_bytes(Thread::pending_exception_offset()))); 89 cbz(rscratch1, L); 90 // exception pending => remove activation and forward to exception handler 91 // make sure that the vm_results are cleared 92 if (oop_result1->is_valid()) { 93 str(zr, Address(rthread, JavaThread::vm_result_offset())); 94 } 95 if (metadata_result->is_valid()) { 96 str(zr, Address(rthread, JavaThread::vm_result_2_offset())); 97 } 98 if (frame_size() == no_frame_size) { 99 leave(); 100 far_jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 101 } else if (_stub_id == Runtime1::forward_exception_id) { 102 should_not_reach_here(); 103 } else { 104 far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id))); 105 } 106 bind(L); 107 } 108 // get oop results if there are any and reset the values in the thread 109 if (oop_result1->is_valid()) { 110 get_vm_result(oop_result1, rthread); 111 } 112 if (metadata_result->is_valid()) { 113 get_vm_result_2(metadata_result, rthread); 114 } 115 return call_offset; 116 } 117 118 119 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1) { 120 mov(c_rarg1, arg1); 121 return call_RT(oop_result1, metadata_result, entry, 1); 122 } 123 124 125 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2) { 126 if (c_rarg1 == arg2) { 127 if (c_rarg2 == arg1) { 128 mov(rscratch1, arg1); 129 mov(arg1, arg2); 130 mov(arg2, rscratch1); 131 } else { 132 mov(c_rarg2, arg2); 133 mov(c_rarg1, arg1); 134 } 135 } else { 136 mov(c_rarg1, arg1); 137 mov(c_rarg2, arg2); 138 } 139 return call_RT(oop_result1, metadata_result, entry, 2); 140 } 141 142 143 int StubAssembler::call_RT(Register oop_result1, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) { 144 // if there is any conflict use the stack 145 if (arg1 == c_rarg2 || arg1 == c_rarg3 || 146 arg2 == c_rarg1 || arg2 == c_rarg3 || 147 arg3 == c_rarg1 || arg3 == c_rarg2) { 148 stp(arg3, arg2, Address(pre(sp, -2 * wordSize))); 149 stp(arg1, zr, Address(pre(sp, -2 * wordSize))); 150 ldp(c_rarg1, zr, Address(post(sp, 2 * wordSize))); 151 ldp(c_rarg3, c_rarg2, Address(post(sp, 2 * wordSize))); 152 } else { 153 mov(c_rarg1, arg1); 154 mov(c_rarg2, arg2); 155 mov(c_rarg3, arg3); 156 } 157 return call_RT(oop_result1, metadata_result, entry, 3); 158 } 159 160 // Implementation of StubFrame 161 162 class StubFrame: public StackObj { 163 private: 164 StubAssembler* _sasm; 165 166 public: 167 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments); 168 void load_argument(int offset_in_words, Register reg); 169 170 ~StubFrame(); 171 };; 172 173 174 #define __ _sasm-> 175 176 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments) { 177 _sasm = sasm; 178 __ set_info(name, must_gc_arguments); 179 __ enter(); 180 } 181 182 // load parameters that were stored with LIR_Assembler::store_parameter 183 // Note: offsets for store_parameter and load_argument must match 184 void StubFrame::load_argument(int offset_in_words, Register reg) { 185 // rbp, + 0: link 186 // + 1: return address 187 // + 2: argument with offset 0 188 // + 3: argument with offset 1 189 // + 4: ... 190 191 __ ldr(reg, Address(rfp, (offset_in_words + 2) * BytesPerWord)); 192 } 193 194 195 StubFrame::~StubFrame() { 196 __ leave(); 197 __ ret(lr); 198 } 199 200 #undef __ 201 202 203 // Implementation of Runtime1 204 205 #define __ sasm-> 206 207 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2; 208 209 // Stack layout for saving/restoring all the registers needed during a runtime 210 // call (this includes deoptimization) 211 // Note: note that users of this frame may well have arguments to some runtime 212 // while these values are on the stack. These positions neglect those arguments 213 // but the code in save_live_registers will take the argument count into 214 // account. 215 // 216 217 enum reg_save_layout { 218 reg_save_frame_size = 32 /* float */ + 32 /* integer */ 219 }; 220 221 // Save off registers which might be killed by calls into the runtime. 222 // Tries to smart of about FP registers. In particular we separate 223 // saving and describing the FPU registers for deoptimization since we 224 // have to save the FPU registers twice if we describe them. The 225 // deopt blob is the only thing which needs to describe FPU registers. 226 // In all other cases it should be sufficient to simply save their 227 // current value. 228 229 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs]; 230 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs]; 231 static int reg_save_size_in_words; 232 static int frame_size_in_bytes = -1; 233 234 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) { 235 int frame_size_in_bytes = reg_save_frame_size * BytesPerWord; 236 sasm->set_frame_size(frame_size_in_bytes / BytesPerWord); 237 int frame_size_in_slots = frame_size_in_bytes / sizeof(jint); 238 OopMap* oop_map = new OopMap(frame_size_in_slots, 0); 239 240 for (int i = 0; i < FrameMap::nof_cpu_regs; i++) { 241 Register r = as_Register(i); 242 if (i <= 18 && i != rscratch1->encoding() && i != rscratch2->encoding()) { 243 int sp_offset = cpu_reg_save_offsets[i]; 244 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), 245 r->as_VMReg()); 246 } 247 } 248 249 if (save_fpu_registers) { 250 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { 251 FloatRegister r = as_FloatRegister(i); 252 { 253 int sp_offset = fpu_reg_save_offsets[i]; 254 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), 255 r->as_VMReg()); 256 } 257 } 258 } 259 return oop_map; 260 } 261 262 static OopMap* save_live_registers(StubAssembler* sasm, 263 bool save_fpu_registers = true) { 264 __ block_comment("save_live_registers"); 265 266 __ push(RegSet::range(r0, r29), sp); // integer registers except lr & sp 267 268 if (save_fpu_registers) { 269 for (int i = 30; i >= 0; i -= 2) 270 __ stpd(as_FloatRegister(i), as_FloatRegister(i+1), 271 Address(__ pre(sp, -2 * wordSize))); 272 } else { 273 __ add(sp, sp, -32 * wordSize); 274 } 275 276 return generate_oop_map(sasm, save_fpu_registers); 277 } 278 279 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) { 280 if (restore_fpu_registers) { 281 for (int i = 0; i < 32; i += 2) 282 __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1), 283 Address(__ post(sp, 2 * wordSize))); 284 } else { 285 __ add(sp, sp, 32 * wordSize); 286 } 287 288 __ pop(RegSet::range(r0, r29), sp); 289 } 290 291 static void restore_live_registers_except_r0(StubAssembler* sasm, bool restore_fpu_registers = true) { 292 293 if (restore_fpu_registers) { 294 for (int i = 0; i < 32; i += 2) 295 __ ldpd(as_FloatRegister(i), as_FloatRegister(i+1), 296 Address(__ post(sp, 2 * wordSize))); 297 } else { 298 __ add(sp, sp, 32 * wordSize); 299 } 300 301 __ ldp(zr, r1, Address(__ post(sp, 16))); 302 __ pop(RegSet::range(r2, r29), sp); 303 } 304 305 306 307 void Runtime1::initialize_pd() { 308 int i; 309 int sp_offset = 0; 310 311 // all float registers are saved explicitly 312 assert(FrameMap::nof_fpu_regs == 32, "double registers not handled here"); 313 for (i = 0; i < FrameMap::nof_fpu_regs; i++) { 314 fpu_reg_save_offsets[i] = sp_offset; 315 sp_offset += 2; // SP offsets are in halfwords 316 } 317 318 for (i = 0; i < FrameMap::nof_cpu_regs; i++) { 319 Register r = as_Register(i); 320 cpu_reg_save_offsets[i] = sp_offset; 321 sp_offset += 2; // SP offsets are in halfwords 322 } 323 } 324 325 326 // target: the entry point of the method that creates and posts the exception oop 327 // has_argument: true if the exception needs an argument (passed in rscratch1) 328 329 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) { 330 // make a frame and preserve the caller's caller-save registers 331 OopMap* oop_map = save_live_registers(sasm); 332 int call_offset; 333 if (!has_argument) { 334 call_offset = __ call_RT(noreg, noreg, target); 335 } else { 336 call_offset = __ call_RT(noreg, noreg, target, rscratch1); 337 } 338 OopMapSet* oop_maps = new OopMapSet(); 339 oop_maps->add_gc_map(call_offset, oop_map); 340 341 __ should_not_reach_here(); 342 return oop_maps; 343 } 344 345 346 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) { 347 __ block_comment("generate_handle_exception"); 348 349 // incoming parameters 350 const Register exception_oop = r0; 351 const Register exception_pc = r3; 352 // other registers used in this stub 353 354 // Save registers, if required. 355 OopMapSet* oop_maps = new OopMapSet(); 356 OopMap* oop_map = NULL; 357 switch (id) { 358 case forward_exception_id: 359 // We're handling an exception in the context of a compiled frame. 360 // The registers have been saved in the standard places. Perform 361 // an exception lookup in the caller and dispatch to the handler 362 // if found. Otherwise unwind and dispatch to the callers 363 // exception handler. 364 oop_map = generate_oop_map(sasm, 1 /*thread*/); 365 366 // load and clear pending exception oop into r0 367 __ ldr(exception_oop, Address(rthread, Thread::pending_exception_offset())); 368 __ str(zr, Address(rthread, Thread::pending_exception_offset())); 369 370 // load issuing PC (the return address for this stub) into r3 371 __ ldr(exception_pc, Address(rfp, 1*BytesPerWord)); 372 373 // make sure that the vm_results are cleared (may be unnecessary) 374 __ str(zr, Address(rthread, JavaThread::vm_result_offset())); 375 __ str(zr, Address(rthread, JavaThread::vm_result_2_offset())); 376 break; 377 case handle_exception_nofpu_id: 378 case handle_exception_id: 379 // At this point all registers MAY be live. 380 oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id); 381 break; 382 case handle_exception_from_callee_id: { 383 // At this point all registers except exception oop (r0) and 384 // exception pc (lr) are dead. 385 const int frame_size = 2 /*fp, return address*/; 386 oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0); 387 sasm->set_frame_size(frame_size); 388 break; 389 } 390 default: 391 __ should_not_reach_here(); 392 break; 393 } 394 395 // verify that only r0 and r3 are valid at this time 396 __ invalidate_registers(false, true, true, false, true, true); 397 // verify that r0 contains a valid exception 398 __ verify_not_null_oop(exception_oop); 399 400 #ifdef ASSERT 401 // check that fields in JavaThread for exception oop and issuing pc are 402 // empty before writing to them 403 Label oop_empty; 404 __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset())); 405 __ cbz(rscratch1, oop_empty); 406 __ stop("exception oop already set"); 407 __ bind(oop_empty); 408 409 Label pc_empty; 410 __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset())); 411 __ cbz(rscratch1, pc_empty); 412 __ stop("exception pc already set"); 413 __ bind(pc_empty); 414 #endif 415 416 // save exception oop and issuing pc into JavaThread 417 // (exception handler will load it from here) 418 __ str(exception_oop, Address(rthread, JavaThread::exception_oop_offset())); 419 __ str(exception_pc, Address(rthread, JavaThread::exception_pc_offset())); 420 421 // patch throwing pc into return address (has bci & oop map) 422 __ str(exception_pc, Address(rfp, 1*BytesPerWord)); 423 424 // compute the exception handler. 425 // the exception oop and the throwing pc are read from the fields in JavaThread 426 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc)); 427 oop_maps->add_gc_map(call_offset, oop_map); 428 429 // r0: handler address 430 // will be the deopt blob if nmethod was deoptimized while we looked up 431 // handler regardless of whether handler existed in the nmethod. 432 433 // only r0 is valid at this time, all other registers have been destroyed by the runtime call 434 __ invalidate_registers(false, true, true, true, true, true); 435 436 // patch the return address, this stub will directly return to the exception handler 437 __ str(r0, Address(rfp, 1*BytesPerWord)); 438 439 switch (id) { 440 case forward_exception_id: 441 case handle_exception_nofpu_id: 442 case handle_exception_id: 443 // Restore the registers that were saved at the beginning. 444 restore_live_registers(sasm, id != handle_exception_nofpu_id); 445 break; 446 case handle_exception_from_callee_id: 447 // WIN64_ONLY: No need to add frame::arg_reg_save_area_bytes to SP 448 // since we do a leave anyway. 449 450 // Pop the return address since we are possibly changing SP (restoring from BP). 451 __ leave(); 452 453 // Restore SP from FP if the exception PC is a method handle call site. 454 { 455 Label nope; 456 __ ldrw(rscratch1, Address(rthread, JavaThread::is_method_handle_return_offset())); 457 __ cbzw(rscratch1, nope); 458 __ mov(sp, rfp); 459 __ bind(nope); 460 } 461 462 __ ret(lr); // jump to exception handler 463 break; 464 default: ShouldNotReachHere(); 465 } 466 467 return oop_maps; 468 } 469 470 471 void Runtime1::generate_unwind_exception(StubAssembler *sasm) { 472 // incoming parameters 473 const Register exception_oop = r0; 474 // callee-saved copy of exception_oop during runtime call 475 const Register exception_oop_callee_saved = r19; 476 // other registers used in this stub 477 const Register exception_pc = r3; 478 const Register handler_addr = r1; 479 480 // verify that only r0, is valid at this time 481 __ invalidate_registers(false, true, true, true, true, true); 482 483 #ifdef ASSERT 484 // check that fields in JavaThread for exception oop and issuing pc are empty 485 Label oop_empty; 486 __ ldr(rscratch1, Address(rthread, JavaThread::exception_oop_offset())); 487 __ cbz(rscratch1, oop_empty); 488 __ stop("exception oop must be empty"); 489 __ bind(oop_empty); 490 491 Label pc_empty; 492 __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset())); 493 __ cbz(rscratch1, pc_empty); 494 __ stop("exception pc must be empty"); 495 __ bind(pc_empty); 496 #endif 497 498 // Save our return address because 499 // exception_handler_for_return_address will destroy it. We also 500 // save exception_oop 501 __ stp(lr, exception_oop, Address(__ pre(sp, -2 * wordSize))); 502 503 // search the exception handler address of the caller (using the return address) 504 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), rthread, lr); 505 // r0: exception handler address of the caller 506 507 // Only R0 is valid at this time; all other registers have been 508 // destroyed by the call. 509 __ invalidate_registers(false, true, true, true, false, true); 510 511 // move result of call into correct register 512 __ mov(handler_addr, r0); 513 514 // get throwing pc (= return address). 515 // lr has been destroyed by the call 516 __ ldp(lr, exception_oop, Address(__ post(sp, 2 * wordSize))); 517 __ mov(r3, lr); 518 519 __ verify_not_null_oop(exception_oop); 520 521 { 522 Label foo; 523 __ ldrw(rscratch1, Address(rthread, JavaThread::is_method_handle_return_offset())); 524 __ cbzw(rscratch1, foo); 525 __ mov(sp, rfp); 526 __ bind(foo); 527 } 528 529 // continue at exception handler (return address removed) 530 // note: do *not* remove arguments when unwinding the 531 // activation since the caller assumes having 532 // all arguments on the stack when entering the 533 // runtime to determine the exception handler 534 // (GC happens at call site with arguments!) 535 // r0: exception oop 536 // r3: throwing pc 537 // r1: exception handler 538 __ br(handler_addr); 539 } 540 541 542 543 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) { 544 // use the maximum number of runtime-arguments here because it is difficult to 545 // distinguish each RT-Call. 546 // Note: This number affects also the RT-Call in generate_handle_exception because 547 // the oop-map is shared for all calls. 548 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); 549 assert(deopt_blob != NULL, "deoptimization blob must have been created"); 550 551 OopMap* oop_map = save_live_registers(sasm); 552 553 __ mov(c_rarg0, rthread); 554 Label retaddr; 555 __ set_last_Java_frame(sp, rfp, retaddr, rscratch1); 556 // do the call 557 __ lea(rscratch1, RuntimeAddress(target)); 558 __ blr(rscratch1); 559 __ bind(retaddr); 560 OopMapSet* oop_maps = new OopMapSet(); 561 oop_maps->add_gc_map(__ offset(), oop_map); 562 // verify callee-saved register 563 #ifdef ASSERT 564 { Label L; 565 __ get_thread(rscratch1); 566 __ cmp(rthread, rscratch1); 567 __ br(Assembler::EQ, L); 568 __ stop("StubAssembler::call_RT: rthread not callee saved?"); 569 __ bind(L); 570 } 571 #endif 572 __ reset_last_Java_frame(true); 573 __ maybe_isb(); 574 575 // check for pending exceptions 576 { Label L; 577 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset())); 578 __ cbz(rscratch1, L); 579 // exception pending => remove activation and forward to exception handler 580 581 { Label L1; 582 __ cbnz(r0, L1); // have we deoptimized? 583 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id))); 584 __ bind(L1); 585 } 586 587 // the deopt blob expects exceptions in the special fields of 588 // JavaThread, so copy and clear pending exception. 589 590 // load and clear pending exception 591 __ ldr(r0, Address(rthread, Thread::pending_exception_offset())); 592 __ str(zr, Address(rthread, Thread::pending_exception_offset())); 593 594 // check that there is really a valid exception 595 __ verify_not_null_oop(r0); 596 597 // load throwing pc: this is the return address of the stub 598 __ mov(r3, lr); 599 600 #ifdef ASSERT 601 // check that fields in JavaThread for exception oop and issuing pc are empty 602 Label oop_empty; 603 __ ldr(rscratch1, Address(rthread, Thread::pending_exception_offset())); 604 __ cbz(rscratch1, oop_empty); 605 __ stop("exception oop must be empty"); 606 __ bind(oop_empty); 607 608 Label pc_empty; 609 __ ldr(rscratch1, Address(rthread, JavaThread::exception_pc_offset())); 610 __ cbz(rscratch1, pc_empty); 611 __ stop("exception pc must be empty"); 612 __ bind(pc_empty); 613 #endif 614 615 // store exception oop and throwing pc to JavaThread 616 __ str(r0, Address(rthread, JavaThread::exception_oop_offset())); 617 __ str(r3, Address(rthread, JavaThread::exception_pc_offset())); 618 619 restore_live_registers(sasm); 620 621 __ leave(); 622 623 // Forward the exception directly to deopt blob. We can blow no 624 // registers and must leave throwing pc on the stack. A patch may 625 // have values live in registers so the entry point with the 626 // exception in tls. 627 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_exception_in_tls())); 628 629 __ bind(L); 630 } 631 632 633 // Runtime will return true if the nmethod has been deoptimized during 634 // the patching process. In that case we must do a deopt reexecute instead. 635 636 Label reexecuteEntry, cont; 637 638 __ cbz(r0, cont); // have we deoptimized? 639 640 // Will reexecute. Proper return address is already on the stack we just restore 641 // registers, pop all of our frame but the return address and jump to the deopt blob 642 restore_live_registers(sasm); 643 __ leave(); 644 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); 645 646 __ bind(cont); 647 restore_live_registers(sasm); 648 __ leave(); 649 __ ret(lr); 650 651 return oop_maps; 652 } 653 654 655 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { 656 657 const Register exception_oop = r0; 658 const Register exception_pc = r3; 659 660 // for better readability 661 const bool must_gc_arguments = true; 662 const bool dont_gc_arguments = false; 663 664 // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu 665 bool save_fpu_registers = true; 666 667 // stub code & info for the different stubs 668 OopMapSet* oop_maps = NULL; 669 OopMap* oop_map = NULL; 670 switch (id) { 671 { 672 case forward_exception_id: 673 { 674 oop_maps = generate_handle_exception(id, sasm); 675 __ leave(); 676 __ ret(lr); 677 } 678 break; 679 680 case throw_div0_exception_id: 681 { StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments); 682 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false); 683 } 684 break; 685 686 case throw_null_pointer_exception_id: 687 { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments); 688 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false); 689 } 690 break; 691 692 case new_instance_id: 693 case fast_new_instance_id: 694 case fast_new_instance_init_check_id: 695 { 696 Register klass = r3; // Incoming 697 Register obj = r0; // Result 698 699 if (id == new_instance_id) { 700 __ set_info("new_instance", dont_gc_arguments); 701 } else if (id == fast_new_instance_id) { 702 __ set_info("fast new_instance", dont_gc_arguments); 703 } else { 704 assert(id == fast_new_instance_init_check_id, "bad StubID"); 705 __ set_info("fast new_instance init check", dont_gc_arguments); 706 } 707 708 if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) && 709 UseTLAB && FastTLABRefill) { 710 Label slow_path; 711 Register obj_size = r2; 712 Register t1 = r19; 713 Register t2 = r4; 714 assert_different_registers(klass, obj, obj_size, t1, t2); 715 716 __ stp(r5, r19, Address(__ pre(sp, -2 * wordSize))); 717 718 if (id == fast_new_instance_init_check_id) { 719 // make sure the klass is initialized 720 __ ldrb(rscratch1, Address(klass, InstanceKlass::init_state_offset())); 721 __ cmpw(rscratch1, InstanceKlass::fully_initialized); 722 __ br(Assembler::NE, slow_path); 723 } 724 725 #ifdef ASSERT 726 // assert object can be fast path allocated 727 { 728 Label ok, not_ok; 729 __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset())); 730 __ cmp(obj_size, 0u); 731 __ br(Assembler::LE, not_ok); // make sure it's an instance (LH > 0) 732 __ tstw(obj_size, Klass::_lh_instance_slow_path_bit); 733 __ br(Assembler::EQ, ok); 734 __ bind(not_ok); 735 __ stop("assert(can be fast path allocated)"); 736 __ should_not_reach_here(); 737 __ bind(ok); 738 } 739 #endif // ASSERT 740 741 // if we got here then the TLAB allocation failed, so try 742 // refilling the TLAB or allocating directly from eden. 743 Label retry_tlab, try_eden; 744 __ tlab_refill(retry_tlab, try_eden, slow_path); // does not destroy r3 (klass), returns r5 745 746 __ bind(retry_tlab); 747 748 // get the instance size (size is postive so movl is fine for 64bit) 749 __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset())); 750 751 __ tlab_allocate(obj, obj_size, 0, t1, t2, slow_path); 752 753 __ initialize_object(obj, klass, obj_size, 0, t1, t2); 754 __ verify_oop(obj); 755 __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize))); 756 __ ret(lr); 757 758 __ bind(try_eden); 759 // get the instance size (size is postive so movl is fine for 64bit) 760 __ ldrw(obj_size, Address(klass, Klass::layout_helper_offset())); 761 762 __ eden_allocate(obj, obj_size, 0, t1, slow_path); 763 __ incr_allocated_bytes(rthread, obj_size, 0, rscratch1); 764 765 __ initialize_object(obj, klass, obj_size, 0, t1, t2); 766 __ verify_oop(obj); 767 __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize))); 768 __ ret(lr); 769 770 __ bind(slow_path); 771 __ ldp(r5, r19, Address(__ post(sp, 2 * wordSize))); 772 } 773 774 __ enter(); 775 OopMap* map = save_live_registers(sasm); 776 int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass); 777 oop_maps = new OopMapSet(); 778 oop_maps->add_gc_map(call_offset, map); 779 restore_live_registers_except_r0(sasm); 780 __ verify_oop(obj); 781 __ leave(); 782 __ ret(lr); 783 784 // r0,: new instance 785 } 786 787 break; 788 789 case counter_overflow_id: 790 { 791 Register bci = r0, method = r1; 792 __ enter(); 793 OopMap* map = save_live_registers(sasm); 794 // Retrieve bci 795 __ ldrw(bci, Address(rfp, 2*BytesPerWord)); 796 // And a pointer to the Method* 797 __ ldr(method, Address(rfp, 3*BytesPerWord)); 798 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method); 799 oop_maps = new OopMapSet(); 800 oop_maps->add_gc_map(call_offset, map); 801 restore_live_registers(sasm); 802 __ leave(); 803 __ ret(lr); 804 } 805 break; 806 807 case new_type_array_id: 808 case new_object_array_id: 809 { 810 Register length = r19; // Incoming 811 Register klass = r3; // Incoming 812 Register obj = r0; // Result 813 814 if (id == new_type_array_id) { 815 __ set_info("new_type_array", dont_gc_arguments); 816 } else { 817 __ set_info("new_object_array", dont_gc_arguments); 818 } 819 820 #ifdef ASSERT 821 // assert object type is really an array of the proper kind 822 { 823 Label ok; 824 Register t0 = obj; 825 __ ldrw(t0, Address(klass, Klass::layout_helper_offset())); 826 __ asrw(t0, t0, Klass::_lh_array_tag_shift); 827 int tag = ((id == new_type_array_id) 828 ? Klass::_lh_array_tag_type_value 829 : Klass::_lh_array_tag_obj_value); 830 __ mov(rscratch1, tag); 831 __ cmpw(t0, rscratch1); 832 __ br(Assembler::EQ, ok); 833 __ stop("assert(is an array klass)"); 834 __ should_not_reach_here(); 835 __ bind(ok); 836 } 837 #endif // ASSERT 838 839 if (UseTLAB && FastTLABRefill) { 840 Register arr_size = r4; 841 Register t1 = r2; 842 Register t2 = r5; 843 Label slow_path; 844 assert_different_registers(length, klass, obj, arr_size, t1, t2); 845 846 // check that array length is small enough for fast path. 847 __ mov(rscratch1, C1_MacroAssembler::max_array_allocation_length); 848 __ cmpw(length, rscratch1); 849 __ br(Assembler::HI, slow_path); 850 851 // if we got here then the TLAB allocation failed, so try 852 // refilling the TLAB or allocating directly from eden. 853 Label retry_tlab, try_eden; 854 const Register thread = 855 __ tlab_refill(retry_tlab, try_eden, slow_path); // preserves r19 & r3, returns rthread 856 857 __ bind(retry_tlab); 858 859 // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) 860 // since size is positive ldrw does right thing on 64bit 861 __ ldrw(t1, Address(klass, Klass::layout_helper_offset())); 862 __ lslvw(arr_size, length, t1); 863 __ ubfx(t1, t1, Klass::_lh_header_size_shift, 864 exact_log2(Klass::_lh_header_size_mask + 1)); 865 __ add(arr_size, arr_size, t1); 866 __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up 867 __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask); 868 869 __ tlab_allocate(obj, arr_size, 0, t1, t2, slow_path); // preserves arr_size 870 871 __ initialize_header(obj, klass, length, t1, t2); 872 __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte))); 873 assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); 874 assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise"); 875 __ andr(t1, t1, Klass::_lh_header_size_mask); 876 __ sub(arr_size, arr_size, t1); // body length 877 __ add(t1, t1, obj); // body start 878 __ initialize_body(t1, arr_size, 0, t2); 879 __ membar(Assembler::StoreStore); 880 __ verify_oop(obj); 881 882 __ ret(lr); 883 884 __ bind(try_eden); 885 // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) 886 // since size is positive ldrw does right thing on 64bit 887 __ ldrw(t1, Address(klass, Klass::layout_helper_offset())); 888 // since size is postive movw does right thing on 64bit 889 __ movw(arr_size, length); 890 __ lslvw(arr_size, length, t1); 891 __ ubfx(t1, t1, Klass::_lh_header_size_shift, 892 exact_log2(Klass::_lh_header_size_mask + 1)); 893 __ add(arr_size, arr_size, t1); 894 __ add(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up 895 __ andr(arr_size, arr_size, ~MinObjAlignmentInBytesMask); 896 897 __ eden_allocate(obj, arr_size, 0, t1, slow_path); // preserves arr_size 898 __ incr_allocated_bytes(thread, arr_size, 0, rscratch1); 899 900 __ initialize_header(obj, klass, length, t1, t2); 901 __ ldrb(t1, Address(klass, in_bytes(Klass::layout_helper_offset()) + (Klass::_lh_header_size_shift / BitsPerByte))); 902 assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); 903 assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise"); 904 __ andr(t1, t1, Klass::_lh_header_size_mask); 905 __ sub(arr_size, arr_size, t1); // body length 906 __ add(t1, t1, obj); // body start 907 __ initialize_body(t1, arr_size, 0, t2); 908 __ membar(Assembler::StoreStore); 909 __ verify_oop(obj); 910 911 __ ret(lr); 912 913 __ bind(slow_path); 914 } 915 916 __ enter(); 917 OopMap* map = save_live_registers(sasm); 918 int call_offset; 919 if (id == new_type_array_id) { 920 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length); 921 } else { 922 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length); 923 } 924 925 oop_maps = new OopMapSet(); 926 oop_maps->add_gc_map(call_offset, map); 927 restore_live_registers_except_r0(sasm); 928 929 __ verify_oop(obj); 930 __ leave(); 931 __ ret(lr); 932 933 // r0: new array 934 } 935 break; 936 937 case new_multi_array_id: 938 { StubFrame f(sasm, "new_multi_array", dont_gc_arguments); 939 // r0,: klass 940 // r19,: rank 941 // r2: address of 1st dimension 942 OopMap* map = save_live_registers(sasm); 943 __ mov(c_rarg1, r0); 944 __ mov(c_rarg3, r2); 945 __ mov(c_rarg2, r19); 946 int call_offset = __ call_RT(r0, noreg, CAST_FROM_FN_PTR(address, new_multi_array), r1, r2, r3); 947 948 oop_maps = new OopMapSet(); 949 oop_maps->add_gc_map(call_offset, map); 950 restore_live_registers_except_r0(sasm); 951 952 // r0,: new multi array 953 __ verify_oop(r0); 954 } 955 break; 956 957 case register_finalizer_id: 958 { 959 __ set_info("register_finalizer", dont_gc_arguments); 960 961 // This is called via call_runtime so the arguments 962 // will be place in C abi locations 963 964 __ verify_oop(c_rarg0); 965 966 // load the klass and check the has finalizer flag 967 Label register_finalizer; 968 Register t = r5; 969 __ load_klass(t, r0); 970 __ ldrw(t, Address(t, Klass::access_flags_offset())); 971 __ tst(t, JVM_ACC_HAS_FINALIZER); 972 __ br(Assembler::NE, register_finalizer); 973 __ ret(lr); 974 975 __ bind(register_finalizer); 976 __ enter(); 977 OopMap* oop_map = save_live_registers(sasm); 978 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), r0); 979 oop_maps = new OopMapSet(); 980 oop_maps->add_gc_map(call_offset, oop_map); 981 982 // Now restore all the live registers 983 restore_live_registers(sasm); 984 985 __ leave(); 986 __ ret(lr); 987 } 988 break; 989 990 case throw_class_cast_exception_id: 991 { StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments); 992 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true); 993 } 994 break; 995 996 case throw_incompatible_class_change_error_id: 997 { StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments); 998 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false); 999 } 1000 break; 1001 1002 case slow_subtype_check_id: 1003 { 1004 // Typical calling sequence: 1005 // __ push(klass_RInfo); // object klass or other subclass 1006 // __ push(sup_k_RInfo); // array element klass or other superclass 1007 // __ bl(slow_subtype_check); 1008 // Note that the subclass is pushed first, and is therefore deepest. 1009 enum layout { 1010 r0_off, r0_off_hi, 1011 r2_off, r2_off_hi, 1012 r4_off, r4_off_hi, 1013 r5_off, r5_off_hi, 1014 sup_k_off, sup_k_off_hi, 1015 klass_off, klass_off_hi, 1016 framesize, 1017 result_off = sup_k_off 1018 }; 1019 1020 __ set_info("slow_subtype_check", dont_gc_arguments); 1021 __ push(RegSet::of(r0, r2, r4, r5), sp); 1022 1023 // This is called by pushing args and not with C abi 1024 // __ ldr(r4, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // subclass 1025 // __ ldr(r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // superclass 1026 1027 __ ldp(r4, r0, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); 1028 1029 Label miss; 1030 __ check_klass_subtype_slow_path(r4, r0, r2, r5, NULL, &miss); 1031 1032 // fallthrough on success: 1033 __ mov(rscratch1, 1); 1034 __ str(rscratch1, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result 1035 __ pop(RegSet::of(r0, r2, r4, r5), sp); 1036 __ ret(lr); 1037 1038 __ bind(miss); 1039 __ str(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result 1040 __ pop(RegSet::of(r0, r2, r4, r5), sp); 1041 __ ret(lr); 1042 } 1043 break; 1044 1045 case monitorenter_nofpu_id: 1046 save_fpu_registers = false; 1047 // fall through 1048 case monitorenter_id: 1049 { 1050 StubFrame f(sasm, "monitorenter", dont_gc_arguments); 1051 OopMap* map = save_live_registers(sasm, save_fpu_registers); 1052 1053 // Called with store_parameter and not C abi 1054 1055 f.load_argument(1, r0); // r0,: object 1056 f.load_argument(0, r1); // r1,: lock address 1057 1058 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), r0, r1); 1059 1060 oop_maps = new OopMapSet(); 1061 oop_maps->add_gc_map(call_offset, map); 1062 restore_live_registers(sasm, save_fpu_registers); 1063 } 1064 break; 1065 1066 case monitorexit_nofpu_id: 1067 save_fpu_registers = false; 1068 // fall through 1069 case monitorexit_id: 1070 { 1071 StubFrame f(sasm, "monitorexit", dont_gc_arguments); 1072 OopMap* map = save_live_registers(sasm, save_fpu_registers); 1073 1074 // Called with store_parameter and not C abi 1075 1076 f.load_argument(0, r0); // r0,: lock address 1077 1078 // note: really a leaf routine but must setup last java sp 1079 // => use call_RT for now (speed can be improved by 1080 // doing last java sp setup manually) 1081 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), r0); 1082 1083 oop_maps = new OopMapSet(); 1084 oop_maps->add_gc_map(call_offset, map); 1085 restore_live_registers(sasm, save_fpu_registers); 1086 } 1087 break; 1088 1089 case deoptimize_id: 1090 { 1091 StubFrame f(sasm, "deoptimize", dont_gc_arguments); 1092 OopMap* oop_map = save_live_registers(sasm); 1093 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize)); 1094 oop_maps = new OopMapSet(); 1095 oop_maps->add_gc_map(call_offset, oop_map); 1096 restore_live_registers(sasm); 1097 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); 1098 assert(deopt_blob != NULL, "deoptimization blob must have been created"); 1099 __ leave(); 1100 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); 1101 } 1102 break; 1103 1104 case throw_range_check_failed_id: 1105 { StubFrame f(sasm, "range_check_failed", dont_gc_arguments); 1106 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true); 1107 } 1108 break; 1109 1110 case unwind_exception_id: 1111 { __ set_info("unwind_exception", dont_gc_arguments); 1112 // note: no stubframe since we are about to leave the current 1113 // activation and we are calling a leaf VM function only. 1114 generate_unwind_exception(sasm); 1115 } 1116 break; 1117 1118 case access_field_patching_id: 1119 { StubFrame f(sasm, "access_field_patching", dont_gc_arguments); 1120 // we should set up register map 1121 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching)); 1122 } 1123 break; 1124 1125 case load_klass_patching_id: 1126 { StubFrame f(sasm, "load_klass_patching", dont_gc_arguments); 1127 // we should set up register map 1128 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching)); 1129 } 1130 break; 1131 1132 case load_mirror_patching_id: 1133 { StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments); 1134 // we should set up register map 1135 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching)); 1136 } 1137 break; 1138 1139 case load_appendix_patching_id: 1140 { StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments); 1141 // we should set up register map 1142 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching)); 1143 } 1144 break; 1145 1146 case handle_exception_nofpu_id: 1147 case handle_exception_id: 1148 { StubFrame f(sasm, "handle_exception", dont_gc_arguments); 1149 oop_maps = generate_handle_exception(id, sasm); 1150 } 1151 break; 1152 1153 case handle_exception_from_callee_id: 1154 { StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments); 1155 oop_maps = generate_handle_exception(id, sasm); 1156 } 1157 break; 1158 1159 case throw_index_exception_id: 1160 { StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments); 1161 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true); 1162 } 1163 break; 1164 1165 case throw_array_store_exception_id: 1166 { StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments); 1167 // tos + 0: link 1168 // + 1: return address 1169 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true); 1170 } 1171 break; 1172 1173 #if INCLUDE_ALL_GCS 1174 1175 case g1_pre_barrier_slow_id: 1176 { 1177 StubFrame f(sasm, "g1_pre_barrier", dont_gc_arguments); 1178 // arg0 : previous value of memory 1179 1180 BarrierSet* bs = Universe::heap()->barrier_set(); 1181 if (bs->kind() != BarrierSet::G1SATBCTLogging) { 1182 __ mov(r0, (int)id); 1183 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0); 1184 __ should_not_reach_here(); 1185 break; 1186 } 1187 1188 const Register pre_val = r0; 1189 const Register thread = rthread; 1190 const Register tmp = rscratch1; 1191 1192 Address in_progress(thread, in_bytes(JavaThread::satb_mark_queue_offset() + 1193 PtrQueue::byte_offset_of_active())); 1194 1195 Address queue_index(thread, in_bytes(JavaThread::satb_mark_queue_offset() + 1196 PtrQueue::byte_offset_of_index())); 1197 Address buffer(thread, in_bytes(JavaThread::satb_mark_queue_offset() + 1198 PtrQueue::byte_offset_of_buf())); 1199 1200 Label done; 1201 Label runtime; 1202 1203 // Can we store original value in the thread's buffer? 1204 __ ldr(tmp, queue_index); 1205 __ cbz(tmp, runtime); 1206 1207 __ sub(tmp, tmp, wordSize); 1208 __ str(tmp, queue_index); 1209 __ ldr(rscratch2, buffer); 1210 __ add(tmp, tmp, rscratch2); 1211 f.load_argument(0, rscratch2); 1212 __ str(rscratch2, Address(tmp, 0)); 1213 __ b(done); 1214 1215 __ bind(runtime); 1216 __ push_call_clobbered_registers(); 1217 f.load_argument(0, pre_val); 1218 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_pre), pre_val, thread); 1219 __ pop_call_clobbered_registers(); 1220 __ bind(done); 1221 } 1222 break; 1223 case g1_post_barrier_slow_id: 1224 { 1225 StubFrame f(sasm, "g1_post_barrier", dont_gc_arguments); 1226 1227 // arg0: store_address 1228 Address store_addr(rfp, 2*BytesPerWord); 1229 1230 BarrierSet* bs = Universe::heap()->barrier_set(); 1231 CardTableModRefBS* ct = (CardTableModRefBS*)bs; 1232 assert(sizeof(*ct->byte_map_base) == sizeof(jbyte), "adjust this code"); 1233 1234 Label done; 1235 Label runtime; 1236 1237 // At this point we know new_value is non-NULL and the new_value crosses regions. 1238 // Must check to see if card is already dirty 1239 1240 const Register thread = rthread; 1241 1242 Address queue_index(thread, in_bytes(JavaThread::dirty_card_queue_offset() + 1243 PtrQueue::byte_offset_of_index())); 1244 Address buffer(thread, in_bytes(JavaThread::dirty_card_queue_offset() + 1245 PtrQueue::byte_offset_of_buf())); 1246 1247 const Register card_offset = rscratch2; 1248 // LR is free here, so we can use it to hold the byte_map_base. 1249 const Register byte_map_base = lr; 1250 1251 assert_different_registers(card_offset, byte_map_base, rscratch1); 1252 1253 f.load_argument(0, card_offset); 1254 __ lsr(card_offset, card_offset, CardTableModRefBS::card_shift); 1255 __ load_byte_map_base(byte_map_base); 1256 __ ldrb(rscratch1, Address(byte_map_base, card_offset)); 1257 __ cmpw(rscratch1, (int)G1SATBCardTableModRefBS::g1_young_card_val()); 1258 __ br(Assembler::EQ, done); 1259 1260 assert((int)CardTableModRefBS::dirty_card_val() == 0, "must be 0"); 1261 1262 __ membar(Assembler::StoreLoad); 1263 __ ldrb(rscratch1, Address(byte_map_base, card_offset)); 1264 __ cbzw(rscratch1, done); 1265 1266 // storing region crossing non-NULL, card is clean. 1267 // dirty card and log. 1268 __ strb(zr, Address(byte_map_base, card_offset)); 1269 1270 // Convert card offset into an address in card_addr 1271 Register card_addr = card_offset; 1272 __ add(card_addr, byte_map_base, card_addr); 1273 1274 __ ldr(rscratch1, queue_index); 1275 __ cbz(rscratch1, runtime); 1276 __ sub(rscratch1, rscratch1, wordSize); 1277 __ str(rscratch1, queue_index); 1278 1279 // Reuse LR to hold buffer_addr 1280 const Register buffer_addr = lr; 1281 1282 __ ldr(buffer_addr, buffer); 1283 __ str(card_addr, Address(buffer_addr, rscratch1)); 1284 __ b(done); 1285 1286 __ bind(runtime); 1287 __ push_call_clobbered_registers(); 1288 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::g1_wb_post), card_addr, thread); 1289 __ pop_call_clobbered_registers(); 1290 __ bind(done); 1291 1292 } 1293 break; 1294 #endif 1295 1296 case predicate_failed_trap_id: 1297 { 1298 StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments); 1299 1300 OopMap* map = save_live_registers(sasm); 1301 1302 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap)); 1303 oop_maps = new OopMapSet(); 1304 oop_maps->add_gc_map(call_offset, map); 1305 restore_live_registers(sasm); 1306 __ leave(); 1307 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); 1308 assert(deopt_blob != NULL, "deoptimization blob must have been created"); 1309 1310 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); 1311 } 1312 break; 1313 1314 case dtrace_object_alloc_id: 1315 { // c_rarg0: object 1316 StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments); 1317 save_live_registers(sasm); 1318 1319 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_object_alloc), c_rarg0); 1320 1321 restore_live_registers(sasm); 1322 } 1323 break; 1324 1325 default: 1326 { StubFrame f(sasm, "unimplemented entry", dont_gc_arguments); 1327 __ mov(r0, (int)id); 1328 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), r0); 1329 __ should_not_reach_here(); 1330 } 1331 break; 1332 } 1333 } 1334 return oop_maps; 1335 } 1336 1337 #undef __ 1338 1339 // Simple helper to see if the caller of a runtime stub which 1340 // entered the VM has been deoptimized 1341 1342 static bool caller_is_deopted() { 1343 JavaThread* thread = JavaThread::current(); 1344 RegisterMap reg_map(thread, false); 1345 frame runtime_frame = thread->last_frame(); 1346 frame caller_frame = runtime_frame.sender(®_map); 1347 assert(caller_frame.is_compiled_frame(), "must be compiled"); 1348 return caller_frame.is_deoptimized_frame(); 1349 } 1350 1351 JRT_ENTRY(void, Runtime1::patch_code_aarch64(JavaThread* thread, Runtime1::StubID stub_id )) 1352 { 1353 RegisterMap reg_map(thread, false); 1354 1355 NOT_PRODUCT(_patch_code_slowcase_cnt++;) 1356 // According to the ARMv8 ARM, "Concurrent modification and 1357 // execution of instructions can lead to the resulting instruction 1358 // performing any behavior that can be achieved by executing any 1359 // sequence of instructions that can be executed from the same 1360 // Exception level, except where the instruction before 1361 // modification and the instruction after modification is a B, BL, 1362 // NOP, BKPT, SVC, HVC, or SMC instruction." 1363 // 1364 // This effectively makes the games we play when patching 1365 // impossible, so when we come across an access that needs 1366 // patching we must deoptimize. 1367 1368 if (TracePatching) { 1369 tty->print_cr("Deoptimizing because patch is needed"); 1370 } 1371 1372 frame runtime_frame = thread->last_frame(); 1373 frame caller_frame = runtime_frame.sender(®_map); 1374 1375 // It's possible the nmethod was invalidated in the last 1376 // safepoint, but if it's still alive then make it not_entrant. 1377 nmethod* nm = CodeCache::find_nmethod(caller_frame.pc()); 1378 if (nm != NULL) { 1379 nm->make_not_entrant(); 1380 } 1381 1382 Deoptimization::deoptimize_frame(thread, caller_frame.id()); 1383 1384 // Return to the now deoptimized frame. 1385 } 1386 JRT_END 1387 1388 int Runtime1::access_field_patching(JavaThread* thread) { 1389 // 1390 // NOTE: we are still in Java 1391 // 1392 Thread* THREAD = thread; 1393 debug_only(NoHandleMark nhm;) 1394 { 1395 // Enter VM mode 1396 1397 ResetNoHandleMark rnhm; 1398 patch_code_aarch64(thread, access_field_patching_id); 1399 } 1400 // Back in JAVA, use no oops DON'T safepoint 1401 1402 // Return true if calling code is deoptimized 1403 1404 return caller_is_deopted(); 1405 JRT_END 1406 1407 1408 int Runtime1::move_mirror_patching(JavaThread* thread) { 1409 // 1410 // NOTE: we are still in Java 1411 // 1412 Thread* THREAD = thread; 1413 debug_only(NoHandleMark nhm;) 1414 { 1415 // Enter VM mode 1416 1417 ResetNoHandleMark rnhm; 1418 patch_code_aarch64(thread, load_mirror_patching_id); 1419 } 1420 // Back in JAVA, use no oops DON'T safepoint 1421 1422 // Return true if calling code is deoptimized 1423 1424 return caller_is_deopted(); 1425 } 1426 1427 int Runtime1::move_appendix_patching(JavaThread* thread) { 1428 // 1429 // NOTE: we are still in Java 1430 // 1431 Thread* THREAD = thread; 1432 debug_only(NoHandleMark nhm;) 1433 { 1434 // Enter VM mode 1435 1436 ResetNoHandleMark rnhm; 1437 patch_code_aarch64(thread, load_appendix_patching_id); 1438 } 1439 // Back in JAVA, use no oops DON'T safepoint 1440 1441 // Return true if calling code is deoptimized 1442 1443 return caller_is_deopted(); 1444 } 1445 1446 int Runtime1::move_klass_patching(JavaThread* thread) { 1447 // 1448 // NOTE: we are still in Java 1449 // 1450 Thread* THREAD = thread; 1451 debug_only(NoHandleMark nhm;) 1452 { 1453 // Enter VM mode 1454 1455 ResetNoHandleMark rnhm; 1456 patch_code_aarch64(thread, load_klass_patching_id); 1457 } 1458 // Back in JAVA, use no oops DON'T safepoint 1459 1460 // Return true if calling code is deoptimized 1461 1462 return caller_is_deopted(); 1463 } 1464 1465 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }