1 /* 2 * Copyright (c) 1999, 2023, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, Red Hat Inc. All rights reserved. 4 * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 #include "asm/assembler.hpp" 29 #include "c1/c1_CodeStubs.hpp" 30 #include "c1/c1_Defs.hpp" 31 #include "c1/c1_MacroAssembler.hpp" 32 #include "c1/c1_Runtime1.hpp" 33 #include "compiler/disassembler.hpp" 34 #include "compiler/oopMap.hpp" 35 #include "gc/shared/cardTable.hpp" 36 #include "gc/shared/cardTableBarrierSet.hpp" 37 #include "interpreter/interpreter.hpp" 38 #include "memory/universe.hpp" 39 #include "nativeInst_riscv.hpp" 40 #include "oops/oop.inline.hpp" 41 #include "prims/jvmtiExport.hpp" 42 #include "register_riscv.hpp" 43 #include "runtime/sharedRuntime.hpp" 44 #include "runtime/signature.hpp" 45 #include "runtime/stubRoutines.hpp" 46 #include "runtime/vframe.hpp" 47 #include "runtime/vframeArray.hpp" 48 #include "utilities/powerOfTwo.hpp" 49 #include "vmreg_riscv.inline.hpp" 50 51 52 // Implementation of StubAssembler 53 54 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, int args_size) { 55 // setup registers 56 assert(!(oop_result->is_valid() || metadata_result->is_valid()) || oop_result != metadata_result, 57 "registers must be different"); 58 assert(oop_result != xthread && metadata_result != xthread, "registers must be different"); 59 assert(args_size >= 0, "illegal args_size"); 60 bool align_stack = false; 61 62 mv(c_rarg0, xthread); 63 set_num_rt_args(0); // Nothing on stack 64 65 Label retaddr; 66 set_last_Java_frame(sp, fp, retaddr, t0); 67 68 // do the call 69 rt_call(entry); 70 bind(retaddr); 71 int call_offset = offset(); 72 // verify callee-saved register 73 #ifdef ASSERT 74 push_reg(x10, sp); 75 { Label L; 76 get_thread(x10); 77 beq(xthread, x10, L); 78 stop("StubAssembler::call_RT: xthread not callee saved?"); 79 bind(L); 80 } 81 pop_reg(x10, sp); 82 #endif 83 reset_last_Java_frame(true); 84 85 // check for pending exceptions 86 { Label L; 87 // check for pending exceptions (java_thread is set upon return) 88 ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset()))); 89 beqz(t0, L); 90 // exception pending => remove activation and forward to exception handler 91 // make sure that the vm_results are cleared 92 if (oop_result->is_valid()) { 93 sd(zr, Address(xthread, JavaThread::vm_result_offset())); 94 } 95 if (metadata_result->is_valid()) { 96 sd(zr, Address(xthread, JavaThread::vm_result_2_offset())); 97 } 98 if (frame_size() == no_frame_size) { 99 leave(); 100 far_jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 101 } else if (_stub_id == Runtime1::forward_exception_id) { 102 should_not_reach_here(); 103 } else { 104 far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id))); 105 } 106 bind(L); 107 } 108 // get oop results if there are any and reset the values in the thread 109 if (oop_result->is_valid()) { 110 get_vm_result(oop_result, xthread); 111 } 112 if (metadata_result->is_valid()) { 113 get_vm_result_2(metadata_result, xthread); 114 } 115 return call_offset; 116 } 117 118 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1) { 119 mv(c_rarg1, arg1); 120 return call_RT(oop_result, metadata_result, entry, 1); 121 } 122 123 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1, Register arg2) { 124 const int arg_num = 2; 125 if (c_rarg1 == arg2) { 126 if (c_rarg2 == arg1) { 127 xorr(arg1, arg1, arg2); 128 xorr(arg2, arg1, arg2); 129 xorr(arg1, arg1, arg2); 130 } else { 131 mv(c_rarg2, arg2); 132 mv(c_rarg1, arg1); 133 } 134 } else { 135 mv(c_rarg1, arg1); 136 mv(c_rarg2, arg2); 137 } 138 return call_RT(oop_result, metadata_result, entry, arg_num); 139 } 140 141 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) { 142 const int arg_num = 3; 143 // if there is any conflict use the stack 144 if (arg1 == c_rarg2 || arg1 == c_rarg3 || 145 arg2 == c_rarg1 || arg2 == c_rarg3 || 146 arg3 == c_rarg1 || arg3 == c_rarg2) { 147 const int arg1_sp_offset = 0; 148 const int arg2_sp_offset = 1; 149 const int arg3_sp_offset = 2; 150 addi(sp, sp, -(arg_num + 1) * wordSize); 151 sd(arg1, Address(sp, arg1_sp_offset * wordSize)); 152 sd(arg2, Address(sp, arg2_sp_offset * wordSize)); 153 sd(arg3, Address(sp, arg3_sp_offset * wordSize)); 154 155 ld(c_rarg1, Address(sp, arg1_sp_offset * wordSize)); 156 ld(c_rarg2, Address(sp, arg2_sp_offset * wordSize)); 157 ld(c_rarg3, Address(sp, arg3_sp_offset * wordSize)); 158 addi(sp, sp, (arg_num + 1) * wordSize); 159 } else { 160 mv(c_rarg1, arg1); 161 mv(c_rarg2, arg2); 162 mv(c_rarg3, arg3); 163 } 164 return call_RT(oop_result, metadata_result, entry, arg_num); 165 } 166 167 enum return_state_t { 168 does_not_return, requires_return 169 }; 170 171 // Implementation of StubFrame 172 173 class StubFrame: public StackObj { 174 private: 175 StubAssembler* _sasm; 176 bool _return_state; 177 178 public: 179 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state=requires_return); 180 void load_argument(int offset_in_words, Register reg); 181 182 ~StubFrame(); 183 };; 184 185 void StubAssembler::prologue(const char* name, bool must_gc_arguments) { 186 set_info(name, must_gc_arguments); 187 enter(); 188 } 189 190 void StubAssembler::epilogue(bool use_pop) { 191 leave(); 192 ret(); 193 } 194 195 #define __ _sasm-> 196 197 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state) { 198 _sasm = sasm; 199 _return_state = return_state; 200 __ prologue(name, must_gc_arguments); 201 } 202 203 // load parameters that were stored with LIR_Assembler::store_parameter 204 // Note: offsets for store_parameter and load_argument must match 205 void StubFrame::load_argument(int offset_in_words, Register reg) { 206 __ load_parameter(offset_in_words, reg); 207 } 208 209 210 StubFrame::~StubFrame() { 211 if (_return_state == requires_return) { 212 __ epilogue(); 213 } else { 214 __ should_not_reach_here(); 215 } 216 _sasm = nullptr; 217 } 218 219 #undef __ 220 221 222 // Implementation of Runtime1 223 224 #define __ sasm-> 225 226 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2; 227 228 // Stack layout for saving/restoring all the registers needed during a runtime 229 // call (this includes deoptimization) 230 // Note: note that users of this frame may well have arguments to some runtime 231 // while these values are on the stack. These positions neglect those arguments 232 // but the code in save_live_registers will take the argument count into 233 // account. 234 // 235 236 enum reg_save_layout { 237 reg_save_frame_size = 32 /* float */ + 30 /* integer excluding x3, x4 */ 238 }; 239 240 // Save off registers which might be killed by calls into the runtime. 241 // Tries to smart of about FPU registers. In particular we separate 242 // saving and describing the FPU registers for deoptimization since we 243 // have to save the FPU registers twice if we describe them. The 244 // deopt blob is the only thing which needs to describe FPU registers. 245 // In all other cases it should be sufficient to simply save their 246 // current value. 247 248 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs]; 249 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs]; 250 251 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) { 252 int frame_size_in_bytes = reg_save_frame_size * BytesPerWord; 253 sasm->set_frame_size(frame_size_in_bytes / BytesPerWord); 254 int frame_size_in_slots = frame_size_in_bytes / sizeof(jint); 255 OopMap* oop_map = new OopMap(frame_size_in_slots, 0); 256 assert_cond(oop_map != nullptr); 257 258 // caller save registers only, see FrameMap::initialize 259 // in c1_FrameMap_riscv.cpp for detail. 260 const static Register caller_save_cpu_regs[FrameMap::max_nof_caller_save_cpu_regs] = { 261 x7, x10, x11, x12, x13, x14, x15, x16, x17, x28, x29, x30, x31 262 }; 263 264 for (int i = 0; i < FrameMap::max_nof_caller_save_cpu_regs; i++) { 265 Register r = caller_save_cpu_regs[i]; 266 int sp_offset = cpu_reg_save_offsets[r->encoding()]; 267 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), 268 r->as_VMReg()); 269 } 270 271 // fpu_regs 272 if (save_fpu_registers) { 273 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { 274 FloatRegister r = as_FloatRegister(i); 275 int sp_offset = fpu_reg_save_offsets[i]; 276 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), 277 r->as_VMReg()); 278 } 279 } 280 return oop_map; 281 } 282 283 static OopMap* save_live_registers(StubAssembler* sasm, 284 bool save_fpu_registers = true) { 285 __ block_comment("save_live_registers"); 286 287 // if the number of pushed regs is odd, one slot will be reserved for alignment 288 __ push_reg(RegSet::range(x5, x31), sp); // integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4) 289 290 if (save_fpu_registers) { 291 // float registers 292 __ addi(sp, sp, -(FrameMap::nof_fpu_regs * wordSize)); 293 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { 294 __ fsd(as_FloatRegister(i), Address(sp, i * wordSize)); 295 } 296 } else { 297 // we define reg_save_layout = 62 as the fixed frame size, 298 // we should also sub 32 * wordSize to sp when save_fpu_registers == false 299 __ addi(sp, sp, -32 * wordSize); 300 } 301 302 return generate_oop_map(sasm, save_fpu_registers); 303 } 304 305 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) { 306 if (restore_fpu_registers) { 307 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { 308 __ fld(as_FloatRegister(i), Address(sp, i * wordSize)); 309 } 310 __ addi(sp, sp, FrameMap::nof_fpu_regs * wordSize); 311 } else { 312 // we define reg_save_layout = 64 as the fixed frame size, 313 // we should also add 32 * wordSize to sp when save_fpu_registers == false 314 __ addi(sp, sp, 32 * wordSize); 315 } 316 317 // if the number of popped regs is odd, the reserved slot for alignment will be removed 318 __ pop_reg(RegSet::range(x5, x31), sp); // integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4) 319 } 320 321 static void restore_live_registers_except_r10(StubAssembler* sasm, bool restore_fpu_registers = true) { 322 if (restore_fpu_registers) { 323 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { 324 __ fld(as_FloatRegister(i), Address(sp, i * wordSize)); 325 } 326 __ addi(sp, sp, FrameMap::nof_fpu_regs * wordSize); 327 } else { 328 // we define reg_save_layout = 64 as the fixed frame size, 329 // we should also add 32 * wordSize to sp when save_fpu_registers == false 330 __ addi(sp, sp, 32 * wordSize); 331 } 332 333 // pop integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4) & x10 334 // there is one reserved slot for alignment on the stack in save_live_registers(). 335 __ pop_reg(RegSet::range(x5, x9), sp); // pop x5 ~ x9 with the reserved slot for alignment 336 __ pop_reg(RegSet::range(x11, x31), sp); // pop x11 ~ x31; x10 will be automatically skipped here 337 } 338 339 void Runtime1::initialize_pd() { 340 int i = 0; 341 int sp_offset = 0; 342 const int step = 2; // SP offsets are in halfwords 343 344 // all float registers are saved explicitly 345 for (i = 0; i < FrameMap::nof_fpu_regs; i++) { 346 fpu_reg_save_offsets[i] = sp_offset; 347 sp_offset += step; 348 } 349 350 // a slot reserved for stack 16-byte alignment, see MacroAssembler::push_reg 351 sp_offset += step; 352 // we save x5 ~ x31, except x0 ~ x4: loop starts from x5 353 for (i = 5; i < FrameMap::nof_cpu_regs; i++) { 354 cpu_reg_save_offsets[i] = sp_offset; 355 sp_offset += step; 356 } 357 } 358 359 uint Runtime1::runtime_blob_current_thread_offset(frame f) { 360 Unimplemented(); 361 return 0; 362 } 363 364 // target: the entry point of the method that creates and posts the exception oop 365 // has_argument: true if the exception needs arguments (passed in t0 and t1) 366 367 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) { 368 // make a frame and preserve the caller's caller-save registers 369 OopMap* oop_map = save_live_registers(sasm); 370 assert_cond(oop_map != nullptr); 371 int call_offset = 0; 372 if (!has_argument) { 373 call_offset = __ call_RT(noreg, noreg, target); 374 } else { 375 __ mv(c_rarg1, t0); 376 __ mv(c_rarg2, t1); 377 call_offset = __ call_RT(noreg, noreg, target); 378 } 379 OopMapSet* oop_maps = new OopMapSet(); 380 assert_cond(oop_maps != nullptr); 381 oop_maps->add_gc_map(call_offset, oop_map); 382 383 return oop_maps; 384 } 385 386 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) { 387 __ block_comment("generate_handle_exception"); 388 389 // incoming parameters 390 const Register exception_oop = x10; 391 const Register exception_pc = x13; 392 393 OopMapSet* oop_maps = new OopMapSet(); 394 assert_cond(oop_maps != nullptr); 395 OopMap* oop_map = nullptr; 396 397 switch (id) { 398 case forward_exception_id: 399 // We're handling an exception in the context of a compiled frame. 400 // The registers have been saved in the standard places. Perform 401 // an exception lookup in the caller and dispatch to the handler 402 // if found. Otherwise unwind and dispatch to the callers 403 // exception handler. 404 oop_map = generate_oop_map(sasm, 1 /* thread */); 405 406 // load and clear pending exception oop into x10 407 __ ld(exception_oop, Address(xthread, Thread::pending_exception_offset())); 408 __ sd(zr, Address(xthread, Thread::pending_exception_offset())); 409 410 // load issuing PC (the return address for this stub) into x13 411 __ ld(exception_pc, Address(fp, frame::return_addr_offset * BytesPerWord)); 412 413 // make sure that the vm_results are cleared (may be unnecessary) 414 __ sd(zr, Address(xthread, JavaThread::vm_result_offset())); 415 __ sd(zr, Address(xthread, JavaThread::vm_result_2_offset())); 416 break; 417 case handle_exception_nofpu_id: 418 case handle_exception_id: 419 // At this point all registers MAY be live. 420 oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id); 421 break; 422 case handle_exception_from_callee_id: { 423 // At this point all registers except exception oop (x10) and 424 // exception pc (ra) are dead. 425 const int frame_size = 2 /* fp, return address */; 426 oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0); 427 sasm->set_frame_size(frame_size); 428 break; 429 } 430 default: ShouldNotReachHere(); 431 } 432 433 // verify that only x10 and x13 are valid at this time 434 __ invalidate_registers(false, true, true, false, true, true); 435 // verify that x10 contains a valid exception 436 __ verify_not_null_oop(exception_oop); 437 438 #ifdef ASSERT 439 // check that fields in JavaThread for exception oop and issuing pc are 440 // empty before writing to them 441 Label oop_empty; 442 __ ld(t0, Address(xthread, JavaThread::exception_oop_offset())); 443 __ beqz(t0, oop_empty); 444 __ stop("exception oop already set"); 445 __ bind(oop_empty); 446 447 Label pc_empty; 448 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset())); 449 __ beqz(t0, pc_empty); 450 __ stop("exception pc already set"); 451 __ bind(pc_empty); 452 #endif 453 454 // save exception oop and issuing pc into JavaThread 455 // (exception handler will load it from here) 456 __ sd(exception_oop, Address(xthread, JavaThread::exception_oop_offset())); 457 __ sd(exception_pc, Address(xthread, JavaThread::exception_pc_offset())); 458 459 // patch throwing pc into return address (has bci & oop map) 460 __ sd(exception_pc, Address(fp, frame::return_addr_offset * BytesPerWord)); 461 462 // compute the exception handler. 463 // the exception oop and the throwing pc are read from the fields in JavaThread 464 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc)); 465 guarantee(oop_map != nullptr, "null oop_map!"); 466 oop_maps->add_gc_map(call_offset, oop_map); 467 468 // x10: handler address 469 // will be the deopt blob if nmethod was deoptimized while we looked up 470 // handler regardless of whether handler existed in the nmethod. 471 472 // only x10 is valid at this time, all other registers have been destroyed by the runtime call 473 __ invalidate_registers(false, true, true, true, true, true); 474 475 // patch the return address, this stub will directly return to the exception handler 476 __ sd(x10, Address(fp, frame::return_addr_offset * BytesPerWord)); 477 478 switch (id) { 479 case forward_exception_id: 480 case handle_exception_nofpu_id: 481 case handle_exception_id: 482 // Restore the registers that were saved at the beginning. 483 restore_live_registers(sasm, id != handle_exception_nofpu_id); 484 break; 485 case handle_exception_from_callee_id: 486 break; 487 default: ShouldNotReachHere(); 488 } 489 490 return oop_maps; 491 } 492 493 494 void Runtime1::generate_unwind_exception(StubAssembler *sasm) { 495 // incoming parameters 496 const Register exception_oop = x10; 497 // other registers used in this stub 498 const Register handler_addr = x11; 499 500 if (AbortVMOnException) { 501 __ enter(); 502 save_live_registers(sasm); 503 __ call_VM_leaf(CAST_FROM_FN_PTR(address, check_abort_on_vm_exception), x10); 504 restore_live_registers(sasm); 505 __ leave(); 506 } 507 508 // verify that only x10, is valid at this time 509 __ invalidate_registers(false, true, true, true, true, true); 510 511 #ifdef ASSERT 512 // check that fields in JavaThread for exception oop and issuing pc are empty 513 Label oop_empty; 514 __ ld(t0, Address(xthread, JavaThread::exception_oop_offset())); 515 __ beqz(t0, oop_empty); 516 __ stop("exception oop must be empty"); 517 __ bind(oop_empty); 518 519 Label pc_empty; 520 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset())); 521 __ beqz(t0, pc_empty); 522 __ stop("exception pc must be empty"); 523 __ bind(pc_empty); 524 #endif 525 526 // Save our return address because 527 // exception_handler_for_return_address will destroy it. We also 528 // save exception_oop 529 __ addi(sp, sp, -2 * wordSize); 530 __ sd(exception_oop, Address(sp, wordSize)); 531 __ sd(ra, Address(sp)); 532 533 // search the exception handler address of the caller (using the return address) 534 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), xthread, ra); 535 // x10: exception handler address of the caller 536 537 // Only x10 is valid at this time; all other registers have been 538 // destroyed by the call. 539 __ invalidate_registers(false, true, true, true, false, true); 540 541 // move result of call into correct register 542 __ mv(handler_addr, x10); 543 544 // get throwing pc (= return address). 545 // ra has been destroyed by the call 546 __ ld(ra, Address(sp)); 547 __ ld(exception_oop, Address(sp, wordSize)); 548 __ addi(sp, sp, 2 * wordSize); 549 __ mv(x13, ra); 550 551 __ verify_not_null_oop(exception_oop); 552 553 // continue at exception handler (return address removed) 554 // note: do *not* remove arguments when unwinding the 555 // activation since the caller assumes having 556 // all arguments on the stack when entering the 557 // runtime to determine the exception handler 558 // (GC happens at call site with arguments!) 559 // x10: exception oop 560 // x13: throwing pc 561 // x11: exception handler 562 __ jr(handler_addr); 563 } 564 565 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) { 566 // use the maximum number of runtime-arguments here because it is difficult to 567 // distinguish each RT-Call. 568 // Note: This number affects also the RT-Call in generate_handle_exception because 569 // the oop-map is shared for all calls. 570 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); 571 assert(deopt_blob != nullptr, "deoptimization blob must have been created"); 572 573 OopMap* oop_map = save_live_registers(sasm); 574 assert_cond(oop_map != nullptr); 575 576 __ mv(c_rarg0, xthread); 577 Label retaddr; 578 __ set_last_Java_frame(sp, fp, retaddr, t0); 579 // do the call 580 __ rt_call(target); 581 __ bind(retaddr); 582 OopMapSet* oop_maps = new OopMapSet(); 583 assert_cond(oop_maps != nullptr); 584 oop_maps->add_gc_map(__ offset(), oop_map); 585 // verify callee-saved register 586 #ifdef ASSERT 587 { Label L; 588 __ get_thread(t0); 589 __ beq(xthread, t0, L); 590 __ stop("StubAssembler::call_RT: xthread not callee saved?"); 591 __ bind(L); 592 } 593 #endif 594 __ reset_last_Java_frame(true); 595 596 #ifdef ASSERT 597 // Check that fields in JavaThread for exception oop and issuing pc are empty 598 Label oop_empty; 599 __ ld(t0, Address(xthread, Thread::pending_exception_offset())); 600 __ beqz(t0, oop_empty); 601 __ stop("exception oop must be empty"); 602 __ bind(oop_empty); 603 604 Label pc_empty; 605 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset())); 606 __ beqz(t0, pc_empty); 607 __ stop("exception pc must be empty"); 608 __ bind(pc_empty); 609 #endif 610 611 // Runtime will return true if the nmethod has been deoptimized, this is the 612 // expected scenario and anything else is an error. Note that we maintain a 613 // check on the result purely as a defensive measure. 614 Label no_deopt; 615 __ beqz(x10, no_deopt); // Have we deoptimized? 616 617 // Perform a re-execute. The proper return address is already on the stack, 618 // we just need to restore registers, pop all of our frames but the return 619 // address and jump to the deopt blob. 620 621 restore_live_registers(sasm); 622 __ leave(); 623 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); 624 625 __ bind(no_deopt); 626 __ stop("deopt not performed"); 627 628 return oop_maps; 629 } 630 631 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { 632 // for better readability 633 const bool dont_gc_arguments = false; 634 635 // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu 636 bool save_fpu_registers = true; 637 638 // stub code & info for the different stubs 639 OopMapSet* oop_maps = nullptr; 640 switch (id) { 641 { 642 case forward_exception_id: 643 { 644 oop_maps = generate_handle_exception(id, sasm); 645 __ leave(); 646 __ ret(); 647 } 648 break; 649 650 case throw_div0_exception_id: 651 { 652 StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments, does_not_return); 653 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false); 654 } 655 break; 656 657 case throw_null_pointer_exception_id: 658 { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments, does_not_return); 659 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false); 660 } 661 break; 662 663 case new_instance_id: 664 case fast_new_instance_id: 665 case fast_new_instance_init_check_id: 666 { 667 Register klass = x13; // Incoming 668 Register obj = x10; // Result 669 670 if (id == new_instance_id) { 671 __ set_info("new_instance", dont_gc_arguments); 672 } else if (id == fast_new_instance_id) { 673 __ set_info("fast new_instance", dont_gc_arguments); 674 } else { 675 assert(id == fast_new_instance_init_check_id, "bad StubID"); 676 __ set_info("fast new_instance init check", dont_gc_arguments); 677 } 678 679 __ enter(); 680 OopMap* map = save_live_registers(sasm); 681 assert_cond(map != nullptr); 682 int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass); 683 oop_maps = new OopMapSet(); 684 assert_cond(oop_maps != nullptr); 685 oop_maps->add_gc_map(call_offset, map); 686 restore_live_registers_except_r10(sasm); 687 __ verify_oop(obj); 688 __ leave(); 689 __ ret(); 690 691 // x10: new instance 692 } 693 694 break; 695 696 case counter_overflow_id: 697 { 698 Register bci = x10; 699 Register method = x11; 700 __ enter(); 701 OopMap* map = save_live_registers(sasm); 702 assert_cond(map != nullptr); 703 704 const int bci_off = 0; 705 const int method_off = 1; 706 // Retrieve bci 707 __ lw(bci, Address(fp, bci_off * BytesPerWord)); 708 // And a pointer to the Method* 709 __ ld(method, Address(fp, method_off * BytesPerWord)); 710 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method); 711 oop_maps = new OopMapSet(); 712 assert_cond(oop_maps != nullptr); 713 oop_maps->add_gc_map(call_offset, map); 714 restore_live_registers(sasm); 715 __ leave(); 716 __ ret(); 717 } 718 break; 719 720 case new_type_array_id: 721 case new_object_array_id: 722 { 723 Register length = x9; // Incoming 724 Register klass = x13; // Incoming 725 Register obj = x10; // Result 726 727 if (id == new_type_array_id) { 728 __ set_info("new_type_array", dont_gc_arguments); 729 } else { 730 __ set_info("new_object_array", dont_gc_arguments); 731 } 732 733 #ifdef ASSERT 734 // assert object type is really an array of the proper kind 735 { 736 Label ok; 737 Register tmp = obj; 738 __ lwu(tmp, Address(klass, Klass::layout_helper_offset())); 739 __ sraiw(tmp, tmp, Klass::_lh_array_tag_shift); 740 int tag = ((id == new_type_array_id) ? Klass::_lh_array_tag_type_value : Klass::_lh_array_tag_obj_value); 741 __ mv(t0, tag); 742 __ beq(t0, tmp, ok); 743 __ stop("assert(is an array klass)"); 744 __ should_not_reach_here(); 745 __ bind(ok); 746 } 747 #endif // ASSERT 748 749 __ enter(); 750 OopMap* map = save_live_registers(sasm); 751 assert_cond(map != nullptr); 752 int call_offset = 0; 753 if (id == new_type_array_id) { 754 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length); 755 } else { 756 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length); 757 } 758 759 oop_maps = new OopMapSet(); 760 assert_cond(oop_maps != nullptr); 761 oop_maps->add_gc_map(call_offset, map); 762 restore_live_registers_except_r10(sasm); 763 764 __ verify_oop(obj); 765 __ leave(); 766 __ ret(); 767 768 // x10: new array 769 } 770 break; 771 772 case new_multi_array_id: 773 { 774 StubFrame f(sasm, "new_multi_array", dont_gc_arguments); 775 // x10: klass 776 // x9: rank 777 // x12: address of 1st dimension 778 OopMap* map = save_live_registers(sasm); 779 assert_cond(map != nullptr); 780 __ mv(c_rarg1, x10); 781 __ mv(c_rarg3, x12); 782 __ mv(c_rarg2, x9); 783 int call_offset = __ call_RT(x10, noreg, CAST_FROM_FN_PTR(address, new_multi_array), x11, x12, x13); 784 785 oop_maps = new OopMapSet(); 786 assert_cond(oop_maps != nullptr); 787 oop_maps->add_gc_map(call_offset, map); 788 restore_live_registers_except_r10(sasm); 789 790 // x10: new multi array 791 __ verify_oop(x10); 792 } 793 break; 794 795 case register_finalizer_id: 796 { 797 __ set_info("register_finalizer", dont_gc_arguments); 798 799 // This is called via call_runtime so the arguments 800 // will be place in C abi locations 801 __ verify_oop(c_rarg0); 802 803 // load the klass and check the has finalizer flag 804 Label register_finalizer; 805 Register t = x15; 806 __ load_klass(t, x10); 807 __ lwu(t, Address(t, Klass::access_flags_offset())); 808 __ test_bit(t0, t, exact_log2(JVM_ACC_HAS_FINALIZER)); 809 __ bnez(t0, register_finalizer); 810 __ ret(); 811 812 __ bind(register_finalizer); 813 __ enter(); 814 OopMap* oop_map = save_live_registers(sasm); 815 assert_cond(oop_map != nullptr); 816 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), x10); 817 oop_maps = new OopMapSet(); 818 assert_cond(oop_maps != nullptr); 819 oop_maps->add_gc_map(call_offset, oop_map); 820 821 // Now restore all the live registers 822 restore_live_registers(sasm); 823 824 __ leave(); 825 __ ret(); 826 } 827 break; 828 829 case throw_class_cast_exception_id: 830 { 831 StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments, does_not_return); 832 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true); 833 } 834 break; 835 836 case throw_incompatible_class_change_error_id: 837 { 838 StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments, does_not_return); 839 oop_maps = generate_exception_throw(sasm, 840 CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false); 841 } 842 break; 843 844 case slow_subtype_check_id: 845 { 846 // Typical calling sequence: 847 // push klass_RInfo (object klass or other subclass) 848 // push sup_k_RInfo (array element klass or other superclass) 849 // jump to slow_subtype_check 850 // Note that the subclass is pushed first, and is therefore deepest. 851 enum layout { 852 x10_off, x10_off_hi, 853 x12_off, x12_off_hi, 854 x14_off, x14_off_hi, 855 x15_off, x15_off_hi, 856 sup_k_off, sup_k_off_hi, 857 klass_off, klass_off_hi, 858 framesize, 859 result_off = sup_k_off 860 }; 861 862 __ set_info("slow_subtype_check", dont_gc_arguments); 863 __ push_reg(RegSet::of(x10, x12, x14, x15), sp); 864 865 __ ld(x14, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // sub klass 866 __ ld(x10, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // super klass 867 868 Label miss; 869 __ check_klass_subtype_slow_path(x14, x10, x12, x15, nullptr, &miss); 870 871 // fallthrough on success: 872 __ mv(t0, 1); 873 __ sd(t0, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result 874 __ pop_reg(RegSet::of(x10, x12, x14, x15), sp); 875 __ ret(); 876 877 __ bind(miss); 878 __ sd(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result 879 __ pop_reg(RegSet::of(x10, x12, x14, x15), sp); 880 __ ret(); 881 } 882 break; 883 884 case monitorenter_nofpu_id: 885 save_fpu_registers = false; 886 // fall through 887 case monitorenter_id: 888 { 889 StubFrame f(sasm, "monitorenter", dont_gc_arguments); 890 OopMap* map = save_live_registers(sasm, save_fpu_registers); 891 assert_cond(map != nullptr); 892 893 // Called with store_parameter and not C abi 894 f.load_argument(1, x10); // x10: object 895 f.load_argument(0, x11); // x11: lock address 896 897 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), x10, x11); 898 899 oop_maps = new OopMapSet(); 900 assert_cond(oop_maps != nullptr); 901 oop_maps->add_gc_map(call_offset, map); 902 restore_live_registers(sasm, save_fpu_registers); 903 } 904 break; 905 906 case monitorexit_nofpu_id: 907 save_fpu_registers = false; 908 // fall through 909 case monitorexit_id: 910 { 911 StubFrame f(sasm, "monitorexit", dont_gc_arguments); 912 OopMap* map = save_live_registers(sasm, save_fpu_registers); 913 assert_cond(map != nullptr); 914 915 // Called with store_parameter and not C abi 916 f.load_argument(0, x10); // x10: lock address 917 918 // note: really a leaf routine but must setup last java sp 919 // => use call_RT for now (speed can be improved by 920 // doing last java sp setup manually) 921 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), x10); 922 923 oop_maps = new OopMapSet(); 924 assert_cond(oop_maps != nullptr); 925 oop_maps->add_gc_map(call_offset, map); 926 restore_live_registers(sasm, save_fpu_registers); 927 } 928 break; 929 930 case deoptimize_id: 931 { 932 StubFrame f(sasm, "deoptimize", dont_gc_arguments, does_not_return); 933 OopMap* oop_map = save_live_registers(sasm); 934 assert_cond(oop_map != nullptr); 935 f.load_argument(0, c_rarg1); 936 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), c_rarg1); 937 938 oop_maps = new OopMapSet(); 939 assert_cond(oop_maps != nullptr); 940 oop_maps->add_gc_map(call_offset, oop_map); 941 restore_live_registers(sasm); 942 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); 943 assert(deopt_blob != nullptr, "deoptimization blob must have been created"); 944 __ leave(); 945 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); 946 } 947 break; 948 949 case throw_range_check_failed_id: 950 { 951 StubFrame f(sasm, "range_check_failed", dont_gc_arguments, does_not_return); 952 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true); 953 } 954 break; 955 956 case unwind_exception_id: 957 { 958 __ set_info("unwind_exception", dont_gc_arguments); 959 // note: no stubframe since we are about to leave the current 960 // activation and we are calling a leaf VM function only. 961 generate_unwind_exception(sasm); 962 } 963 break; 964 965 case access_field_patching_id: 966 { 967 StubFrame f(sasm, "access_field_patching", dont_gc_arguments, does_not_return); 968 // we should set up register map 969 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching)); 970 } 971 break; 972 973 case load_klass_patching_id: 974 { 975 StubFrame f(sasm, "load_klass_patching", dont_gc_arguments, does_not_return); 976 // we should set up register map 977 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching)); 978 } 979 break; 980 981 case load_mirror_patching_id: 982 { 983 StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments, does_not_return); 984 // we should set up register map 985 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching)); 986 } 987 break; 988 989 case load_appendix_patching_id: 990 { 991 StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments, does_not_return); 992 // we should set up register map 993 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching)); 994 } 995 break; 996 997 case handle_exception_nofpu_id: 998 case handle_exception_id: 999 { 1000 StubFrame f(sasm, "handle_exception", dont_gc_arguments); 1001 oop_maps = generate_handle_exception(id, sasm); 1002 } 1003 break; 1004 1005 case handle_exception_from_callee_id: 1006 { 1007 StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments); 1008 oop_maps = generate_handle_exception(id, sasm); 1009 } 1010 break; 1011 1012 case throw_index_exception_id: 1013 { 1014 StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments, does_not_return); 1015 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true); 1016 } 1017 break; 1018 1019 case throw_array_store_exception_id: 1020 { 1021 StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments, does_not_return); 1022 // tos + 0: link 1023 // + 1: return address 1024 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true); 1025 } 1026 break; 1027 1028 case predicate_failed_trap_id: 1029 { 1030 StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments, does_not_return); 1031 1032 OopMap* map = save_live_registers(sasm); 1033 assert_cond(map != nullptr); 1034 1035 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap)); 1036 oop_maps = new OopMapSet(); 1037 assert_cond(oop_maps != nullptr); 1038 oop_maps->add_gc_map(call_offset, map); 1039 restore_live_registers(sasm); 1040 __ leave(); 1041 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); 1042 assert(deopt_blob != nullptr, "deoptimization blob must have been created"); 1043 1044 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); 1045 } 1046 break; 1047 1048 case dtrace_object_alloc_id: 1049 { // c_rarg0: object 1050 StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments); 1051 save_live_registers(sasm); 1052 1053 __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), c_rarg0); 1054 1055 restore_live_registers(sasm); 1056 } 1057 break; 1058 1059 default: 1060 { 1061 StubFrame f(sasm, "unimplemented entry", dont_gc_arguments, does_not_return); 1062 __ mv(x10, (int)id); 1063 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), x10); 1064 __ should_not_reach_here(); 1065 } 1066 break; 1067 } 1068 } 1069 return oop_maps; 1070 } 1071 1072 #undef __ 1073 1074 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }