1 /* 2 * Copyright (c) 1999, 2020, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, Red Hat Inc. All rights reserved. 4 * Copyright (c) 2020, 2022, Huawei Technologies Co., Ltd. All rights reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 #include "asm/assembler.hpp" 29 #include "c1/c1_CodeStubs.hpp" 30 #include "c1/c1_Defs.hpp" 31 #include "c1/c1_MacroAssembler.hpp" 32 #include "c1/c1_Runtime1.hpp" 33 #include "compiler/disassembler.hpp" 34 #include "compiler/oopMap.hpp" 35 #include "gc/shared/cardTable.hpp" 36 #include "gc/shared/cardTableBarrierSet.hpp" 37 #include "interpreter/interpreter.hpp" 38 #include "memory/universe.hpp" 39 #include "nativeInst_riscv.hpp" 40 #include "oops/compiledICHolder.hpp" 41 #include "oops/oop.inline.hpp" 42 #include "prims/jvmtiExport.hpp" 43 #include "register_riscv.hpp" 44 #include "runtime/sharedRuntime.hpp" 45 #include "runtime/signature.hpp" 46 #include "runtime/stubRoutines.hpp" 47 #include "runtime/vframe.hpp" 48 #include "runtime/vframeArray.hpp" 49 #include "utilities/powerOfTwo.hpp" 50 #include "vmreg_riscv.inline.hpp" 51 52 53 // Implementation of StubAssembler 54 55 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, int args_size) { 56 // setup registers 57 assert(!(oop_result->is_valid() || metadata_result->is_valid()) || oop_result != metadata_result, 58 "registers must be different"); 59 assert(oop_result != xthread && metadata_result != xthread, "registers must be different"); 60 assert(args_size >= 0, "illegal args_size"); 61 bool align_stack = false; 62 63 mv(c_rarg0, xthread); 64 set_num_rt_args(0); // Nothing on stack 65 66 Label retaddr; 67 set_last_Java_frame(sp, fp, retaddr, t0); 68 69 // do the call 70 int32_t off = 0; 71 la_patchable(t0, RuntimeAddress(entry), off); 72 jalr(x1, t0, off); 73 bind(retaddr); 74 int call_offset = offset(); 75 // verify callee-saved register 76 #ifdef ASSERT 77 push_reg(x10, sp); 78 { Label L; 79 get_thread(x10); 80 beq(xthread, x10, L); 81 stop("StubAssembler::call_RT: xthread not callee saved?"); 82 bind(L); 83 } 84 pop_reg(x10, sp); 85 #endif 86 reset_last_Java_frame(true); 87 88 // check for pending exceptions 89 { Label L; 90 // check for pending exceptions (java_thread is set upon return) 91 ld(t0, Address(xthread, in_bytes(Thread::pending_exception_offset()))); 92 beqz(t0, L); 93 // exception pending => remove activation and forward to exception handler 94 // make sure that the vm_results are cleared 95 if (oop_result->is_valid()) { 96 sd(zr, Address(xthread, JavaThread::vm_result_offset())); 97 } 98 if (metadata_result->is_valid()) { 99 sd(zr, Address(xthread, JavaThread::vm_result_2_offset())); 100 } 101 if (frame_size() == no_frame_size) { 102 leave(); 103 far_jump(RuntimeAddress(StubRoutines::forward_exception_entry())); 104 } else if (_stub_id == Runtime1::forward_exception_id) { 105 should_not_reach_here(); 106 } else { 107 far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::forward_exception_id))); 108 } 109 bind(L); 110 } 111 // get oop results if there are any and reset the values in the thread 112 if (oop_result->is_valid()) { 113 get_vm_result(oop_result, xthread); 114 } 115 if (metadata_result->is_valid()) { 116 get_vm_result_2(metadata_result, xthread); 117 } 118 return call_offset; 119 } 120 121 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1) { 122 mv(c_rarg1, arg1); 123 return call_RT(oop_result, metadata_result, entry, 1); 124 } 125 126 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1, Register arg2) { 127 const int arg_num = 2; 128 if (c_rarg1 == arg2) { 129 if (c_rarg2 == arg1) { 130 xorr(arg1, arg1, arg2); 131 xorr(arg2, arg1, arg2); 132 xorr(arg1, arg1, arg2); 133 } else { 134 mv(c_rarg2, arg2); 135 mv(c_rarg1, arg1); 136 } 137 } else { 138 mv(c_rarg1, arg1); 139 mv(c_rarg2, arg2); 140 } 141 return call_RT(oop_result, metadata_result, entry, arg_num); 142 } 143 144 int StubAssembler::call_RT(Register oop_result, Register metadata_result, address entry, Register arg1, Register arg2, Register arg3) { 145 const int arg_num = 3; 146 // if there is any conflict use the stack 147 if (arg1 == c_rarg2 || arg1 == c_rarg3 || 148 arg2 == c_rarg1 || arg2 == c_rarg3 || 149 arg3 == c_rarg1 || arg3 == c_rarg2) { 150 const int arg1_sp_offset = 0; 151 const int arg2_sp_offset = 1; 152 const int arg3_sp_offset = 2; 153 addi(sp, sp, -(arg_num + 1) * wordSize); 154 sd(arg1, Address(sp, arg1_sp_offset * wordSize)); 155 sd(arg2, Address(sp, arg2_sp_offset * wordSize)); 156 sd(arg3, Address(sp, arg3_sp_offset * wordSize)); 157 158 ld(c_rarg1, Address(sp, arg1_sp_offset * wordSize)); 159 ld(c_rarg2, Address(sp, arg2_sp_offset * wordSize)); 160 ld(c_rarg3, Address(sp, arg3_sp_offset * wordSize)); 161 addi(sp, sp, (arg_num + 1) * wordSize); 162 } else { 163 mv(c_rarg1, arg1); 164 mv(c_rarg2, arg2); 165 mv(c_rarg3, arg3); 166 } 167 return call_RT(oop_result, metadata_result, entry, arg_num); 168 } 169 170 enum return_state_t { 171 does_not_return, requires_return 172 }; 173 174 // Implementation of StubFrame 175 176 class StubFrame: public StackObj { 177 private: 178 StubAssembler* _sasm; 179 bool _return_state; 180 181 public: 182 StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state=requires_return); 183 void load_argument(int offset_in_words, Register reg); 184 185 ~StubFrame(); 186 };; 187 188 void StubAssembler::prologue(const char* name, bool must_gc_arguments) { 189 set_info(name, must_gc_arguments); 190 enter(); 191 } 192 193 void StubAssembler::epilogue() { 194 leave(); 195 ret(); 196 } 197 198 #define __ _sasm-> 199 200 StubFrame::StubFrame(StubAssembler* sasm, const char* name, bool must_gc_arguments, return_state_t return_state) { 201 _sasm = sasm; 202 _return_state = return_state; 203 __ prologue(name, must_gc_arguments); 204 } 205 206 // load parameters that were stored with LIR_Assembler::store_parameter 207 // Note: offsets for store_parameter and load_argument must match 208 void StubFrame::load_argument(int offset_in_words, Register reg) { 209 __ load_parameter(offset_in_words, reg); 210 } 211 212 213 StubFrame::~StubFrame() { 214 if (_return_state == requires_return) { 215 __ epilogue(); 216 } else { 217 __ should_not_reach_here(); 218 } 219 _sasm = NULL; 220 } 221 222 #undef __ 223 224 225 // Implementation of Runtime1 226 227 #define __ sasm-> 228 229 const int float_regs_as_doubles_size_in_slots = pd_nof_fpu_regs_frame_map * 2; 230 231 // Stack layout for saving/restoring all the registers needed during a runtime 232 // call (this includes deoptimization) 233 // Note: note that users of this frame may well have arguments to some runtime 234 // while these values are on the stack. These positions neglect those arguments 235 // but the code in save_live_registers will take the argument count into 236 // account. 237 // 238 239 enum reg_save_layout { 240 reg_save_frame_size = 32 /* float */ + 30 /* integer excluding x3, x4 */ 241 }; 242 243 // Save off registers which might be killed by calls into the runtime. 244 // Tries to smart of about FPU registers. In particular we separate 245 // saving and describing the FPU registers for deoptimization since we 246 // have to save the FPU registers twice if we describe them. The 247 // deopt blob is the only thing which needs to describe FPU registers. 248 // In all other cases it should be sufficient to simply save their 249 // current value. 250 251 static int cpu_reg_save_offsets[FrameMap::nof_cpu_regs]; 252 static int fpu_reg_save_offsets[FrameMap::nof_fpu_regs]; 253 254 static OopMap* generate_oop_map(StubAssembler* sasm, bool save_fpu_registers) { 255 int frame_size_in_bytes = reg_save_frame_size * BytesPerWord; 256 sasm->set_frame_size(frame_size_in_bytes / BytesPerWord); 257 int frame_size_in_slots = frame_size_in_bytes / sizeof(jint); 258 OopMap* oop_map = new OopMap(frame_size_in_slots, 0); 259 assert_cond(oop_map != NULL); 260 261 // caller save registers only, see FrameMap::initialize 262 // in c1_FrameMap_riscv.cpp for detail. 263 const static Register caller_save_cpu_regs[FrameMap::max_nof_caller_save_cpu_regs] = { 264 x7, x10, x11, x12, x13, x14, x15, x16, x17, x28, x29, x30, x31 265 }; 266 267 for (int i = 0; i < FrameMap::max_nof_caller_save_cpu_regs; i++) { 268 Register r = caller_save_cpu_regs[i]; 269 int sp_offset = cpu_reg_save_offsets[r->encoding()]; 270 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), 271 r->as_VMReg()); 272 } 273 274 // fpu_regs 275 if (save_fpu_registers) { 276 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { 277 FloatRegister r = as_FloatRegister(i); 278 int sp_offset = fpu_reg_save_offsets[i]; 279 oop_map->set_callee_saved(VMRegImpl::stack2reg(sp_offset), 280 r->as_VMReg()); 281 } 282 } 283 return oop_map; 284 } 285 286 static OopMap* save_live_registers(StubAssembler* sasm, 287 bool save_fpu_registers = true) { 288 __ block_comment("save_live_registers"); 289 290 // if the number of pushed regs is odd, one slot will be reserved for alignment 291 __ push_reg(RegSet::range(x5, x31), sp); // integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4) 292 293 if (save_fpu_registers) { 294 // float registers 295 __ addi(sp, sp, -(FrameMap::nof_fpu_regs * wordSize)); 296 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { 297 __ fsd(as_FloatRegister(i), Address(sp, i * wordSize)); 298 } 299 } else { 300 // we define reg_save_layout = 62 as the fixed frame size, 301 // we should also sub 32 * wordSize to sp when save_fpu_registers == false 302 __ addi(sp, sp, -32 * wordSize); 303 } 304 305 return generate_oop_map(sasm, save_fpu_registers); 306 } 307 308 static void restore_live_registers(StubAssembler* sasm, bool restore_fpu_registers = true) { 309 if (restore_fpu_registers) { 310 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { 311 __ fld(as_FloatRegister(i), Address(sp, i * wordSize)); 312 } 313 __ addi(sp, sp, FrameMap::nof_fpu_regs * wordSize); 314 } else { 315 // we define reg_save_layout = 64 as the fixed frame size, 316 // we should also add 32 * wordSize to sp when save_fpu_registers == false 317 __ addi(sp, sp, 32 * wordSize); 318 } 319 320 // if the number of popped regs is odd, the reserved slot for alignment will be removed 321 __ pop_reg(RegSet::range(x5, x31), sp); // integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4) 322 } 323 324 static void restore_live_registers_except_r10(StubAssembler* sasm, bool restore_fpu_registers = true) { 325 if (restore_fpu_registers) { 326 for (int i = 0; i < FrameMap::nof_fpu_regs; i++) { 327 __ fld(as_FloatRegister(i), Address(sp, i * wordSize)); 328 } 329 __ addi(sp, sp, FrameMap::nof_fpu_regs * wordSize); 330 } else { 331 // we define reg_save_layout = 64 as the fixed frame size, 332 // we should also add 32 * wordSize to sp when save_fpu_registers == false 333 __ addi(sp, sp, 32 * wordSize); 334 } 335 336 // pop integer registers except ra(x1) & sp(x2) & gp(x3) & tp(x4) & x10 337 // there is one reserved slot for alignment on the stack in save_live_registers(). 338 __ pop_reg(RegSet::range(x5, x9), sp); // pop x5 ~ x9 with the reserved slot for alignment 339 __ pop_reg(RegSet::range(x11, x31), sp); // pop x11 ~ x31; x10 will be automatically skipped here 340 } 341 342 void Runtime1::initialize_pd() { 343 int i = 0; 344 int sp_offset = 0; 345 const int step = 2; // SP offsets are in halfwords 346 347 // all float registers are saved explicitly 348 for (i = 0; i < FrameMap::nof_fpu_regs; i++) { 349 fpu_reg_save_offsets[i] = sp_offset; 350 sp_offset += step; 351 } 352 353 // a slot reserved for stack 16-byte alignment, see MacroAssembler::push_reg 354 sp_offset += step; 355 // we save x5 ~ x31, except x0 ~ x4: loop starts from x5 356 for (i = 5; i < FrameMap::nof_cpu_regs; i++) { 357 cpu_reg_save_offsets[i] = sp_offset; 358 sp_offset += step; 359 } 360 } 361 362 // target: the entry point of the method that creates and posts the exception oop 363 // has_argument: true if the exception needs arguments (passed in t0 and t1) 364 365 OopMapSet* Runtime1::generate_exception_throw(StubAssembler* sasm, address target, bool has_argument) { 366 // make a frame and preserve the caller's caller-save registers 367 OopMap* oop_map = save_live_registers(sasm); 368 assert_cond(oop_map != NULL); 369 int call_offset = 0; 370 if (!has_argument) { 371 call_offset = __ call_RT(noreg, noreg, target); 372 } else { 373 __ mv(c_rarg1, t0); 374 __ mv(c_rarg2, t1); 375 call_offset = __ call_RT(noreg, noreg, target); 376 } 377 OopMapSet* oop_maps = new OopMapSet(); 378 assert_cond(oop_maps != NULL); 379 oop_maps->add_gc_map(call_offset, oop_map); 380 381 return oop_maps; 382 } 383 384 OopMapSet* Runtime1::generate_handle_exception(StubID id, StubAssembler *sasm) { 385 __ block_comment("generate_handle_exception"); 386 387 // incoming parameters 388 const Register exception_oop = x10; 389 const Register exception_pc = x13; 390 391 OopMapSet* oop_maps = new OopMapSet(); 392 assert_cond(oop_maps != NULL); 393 OopMap* oop_map = NULL; 394 395 switch (id) { 396 case forward_exception_id: 397 // We're handling an exception in the context of a compiled frame. 398 // The registers have been saved in the standard places. Perform 399 // an exception lookup in the caller and dispatch to the handler 400 // if found. Otherwise unwind and dispatch to the callers 401 // exception handler. 402 oop_map = generate_oop_map(sasm, 1 /* thread */); 403 404 // load and clear pending exception oop into x10 405 __ ld(exception_oop, Address(xthread, Thread::pending_exception_offset())); 406 __ sd(zr, Address(xthread, Thread::pending_exception_offset())); 407 408 // load issuing PC (the return address for this stub) into x13 409 __ ld(exception_pc, Address(fp, frame::return_addr_offset * BytesPerWord)); 410 411 // make sure that the vm_results are cleared (may be unnecessary) 412 __ sd(zr, Address(xthread, JavaThread::vm_result_offset())); 413 __ sd(zr, Address(xthread, JavaThread::vm_result_2_offset())); 414 break; 415 case handle_exception_nofpu_id: 416 case handle_exception_id: 417 // At this point all registers MAY be live. 418 oop_map = save_live_registers(sasm, id != handle_exception_nofpu_id); 419 break; 420 case handle_exception_from_callee_id: { 421 // At this point all registers except exception oop (x10) and 422 // exception pc (ra) are dead. 423 const int frame_size = 2 /* fp, return address */; 424 oop_map = new OopMap(frame_size * VMRegImpl::slots_per_word, 0); 425 sasm->set_frame_size(frame_size); 426 break; 427 } 428 default: ShouldNotReachHere(); 429 } 430 431 // verify that only x10 and x13 are valid at this time 432 __ invalidate_registers(false, true, true, false, true, true); 433 // verify that x10 contains a valid exception 434 __ verify_not_null_oop(exception_oop); 435 436 #ifdef ASSERT 437 // check that fields in JavaThread for exception oop and issuing pc are 438 // empty before writing to them 439 Label oop_empty; 440 __ ld(t0, Address(xthread, JavaThread::exception_oop_offset())); 441 __ beqz(t0, oop_empty); 442 __ stop("exception oop already set"); 443 __ bind(oop_empty); 444 445 Label pc_empty; 446 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset())); 447 __ beqz(t0, pc_empty); 448 __ stop("exception pc already set"); 449 __ bind(pc_empty); 450 #endif 451 452 // save exception oop and issuing pc into JavaThread 453 // (exception handler will load it from here) 454 __ sd(exception_oop, Address(xthread, JavaThread::exception_oop_offset())); 455 __ sd(exception_pc, Address(xthread, JavaThread::exception_pc_offset())); 456 457 // patch throwing pc into return address (has bci & oop map) 458 __ sd(exception_pc, Address(fp, frame::return_addr_offset * BytesPerWord)); 459 460 // compute the exception handler. 461 // the exception oop and the throwing pc are read from the fields in JavaThread 462 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, exception_handler_for_pc)); 463 guarantee(oop_map != NULL, "NULL oop_map!"); 464 oop_maps->add_gc_map(call_offset, oop_map); 465 466 // x10: handler address 467 // will be the deopt blob if nmethod was deoptimized while we looked up 468 // handler regardless of whether handler existed in the nmethod. 469 470 // only x10 is valid at this time, all other registers have been destroyed by the runtime call 471 __ invalidate_registers(false, true, true, true, true, true); 472 473 // patch the return address, this stub will directly return to the exception handler 474 __ sd(x10, Address(fp, frame::return_addr_offset * BytesPerWord)); 475 476 switch (id) { 477 case forward_exception_id: 478 case handle_exception_nofpu_id: 479 case handle_exception_id: 480 // Restore the registers that were saved at the beginning. 481 restore_live_registers(sasm, id != handle_exception_nofpu_id); 482 break; 483 case handle_exception_from_callee_id: 484 break; 485 default: ShouldNotReachHere(); 486 } 487 488 return oop_maps; 489 } 490 491 492 void Runtime1::generate_unwind_exception(StubAssembler *sasm) { 493 // incoming parameters 494 const Register exception_oop = x10; 495 // other registers used in this stub 496 const Register handler_addr = x11; 497 498 // verify that only x10, is valid at this time 499 __ invalidate_registers(false, true, true, true, true, true); 500 501 #ifdef ASSERT 502 // check that fields in JavaThread for exception oop and issuing pc are empty 503 Label oop_empty; 504 __ ld(t0, Address(xthread, JavaThread::exception_oop_offset())); 505 __ beqz(t0, oop_empty); 506 __ stop("exception oop must be empty"); 507 __ bind(oop_empty); 508 509 Label pc_empty; 510 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset())); 511 __ beqz(t0, pc_empty); 512 __ stop("exception pc must be empty"); 513 __ bind(pc_empty); 514 #endif 515 516 // Save our return address because 517 // exception_handler_for_return_address will destroy it. We also 518 // save exception_oop 519 __ addi(sp, sp, -2 * wordSize); 520 __ sd(exception_oop, Address(sp, wordSize)); 521 __ sd(ra, Address(sp)); 522 523 // search the exception handler address of the caller (using the return address) 524 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::exception_handler_for_return_address), xthread, ra); 525 // x10: exception handler address of the caller 526 527 // Only x10 is valid at this time; all other registers have been 528 // destroyed by the call. 529 __ invalidate_registers(false, true, true, true, false, true); 530 531 // move result of call into correct register 532 __ mv(handler_addr, x10); 533 534 // get throwing pc (= return address). 535 // ra has been destroyed by the call 536 __ ld(ra, Address(sp)); 537 __ ld(exception_oop, Address(sp, wordSize)); 538 __ addi(sp, sp, 2 * wordSize); 539 __ mv(x13, ra); 540 541 __ verify_not_null_oop(exception_oop); 542 543 // continue at exception handler (return address removed) 544 // note: do *not* remove arguments when unwinding the 545 // activation since the caller assumes having 546 // all arguments on the stack when entering the 547 // runtime to determine the exception handler 548 // (GC happens at call site with arguments!) 549 // x10: exception oop 550 // x13: throwing pc 551 // x11: exception handler 552 __ jr(handler_addr); 553 } 554 555 OopMapSet* Runtime1::generate_patching(StubAssembler* sasm, address target) { 556 // use the maximum number of runtime-arguments here because it is difficult to 557 // distinguish each RT-Call. 558 // Note: This number affects also the RT-Call in generate_handle_exception because 559 // the oop-map is shared for all calls. 560 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); 561 assert(deopt_blob != NULL, "deoptimization blob must have been created"); 562 563 OopMap* oop_map = save_live_registers(sasm); 564 assert_cond(oop_map != NULL); 565 566 __ mv(c_rarg0, xthread); 567 Label retaddr; 568 __ set_last_Java_frame(sp, fp, retaddr, t0); 569 // do the call 570 int32_t off = 0; 571 __ la_patchable(t0, RuntimeAddress(target), off); 572 __ jalr(x1, t0, off); 573 __ bind(retaddr); 574 OopMapSet* oop_maps = new OopMapSet(); 575 assert_cond(oop_maps != NULL); 576 oop_maps->add_gc_map(__ offset(), oop_map); 577 // verify callee-saved register 578 #ifdef ASSERT 579 { Label L; 580 __ get_thread(t0); 581 __ beq(xthread, t0, L); 582 __ stop("StubAssembler::call_RT: xthread not callee saved?"); 583 __ bind(L); 584 } 585 #endif 586 __ reset_last_Java_frame(true); 587 588 #ifdef ASSERT 589 // Check that fields in JavaThread for exception oop and issuing pc are empty 590 Label oop_empty; 591 __ ld(t0, Address(xthread, Thread::pending_exception_offset())); 592 __ beqz(t0, oop_empty); 593 __ stop("exception oop must be empty"); 594 __ bind(oop_empty); 595 596 Label pc_empty; 597 __ ld(t0, Address(xthread, JavaThread::exception_pc_offset())); 598 __ beqz(t0, pc_empty); 599 __ stop("exception pc must be empty"); 600 __ bind(pc_empty); 601 #endif 602 603 // Runtime will return true if the nmethod has been deoptimized, this is the 604 // expected scenario and anything else is an error. Note that we maintain a 605 // check on the result purely as a defensive measure. 606 Label no_deopt; 607 __ beqz(x10, no_deopt); // Have we deoptimized? 608 609 // Perform a re-execute. The proper return address is already on the stack, 610 // we just need to restore registers, pop all of our frames but the return 611 // address and jump to the deopt blob. 612 613 restore_live_registers(sasm); 614 __ leave(); 615 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); 616 617 __ bind(no_deopt); 618 __ stop("deopt not performed"); 619 620 return oop_maps; 621 } 622 623 OopMapSet* Runtime1::generate_code_for(StubID id, StubAssembler* sasm) { 624 // for better readability 625 const bool dont_gc_arguments = false; 626 627 // default value; overwritten for some optimized stubs that are called from methods that do not use the fpu 628 bool save_fpu_registers = true; 629 630 // stub code & info for the different stubs 631 OopMapSet* oop_maps = NULL; 632 switch (id) { 633 { 634 case forward_exception_id: 635 { 636 oop_maps = generate_handle_exception(id, sasm); 637 __ leave(); 638 __ ret(); 639 } 640 break; 641 642 case throw_div0_exception_id: 643 { 644 StubFrame f(sasm, "throw_div0_exception", dont_gc_arguments, does_not_return); 645 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_div0_exception), false); 646 } 647 break; 648 649 case throw_null_pointer_exception_id: 650 { StubFrame f(sasm, "throw_null_pointer_exception", dont_gc_arguments, does_not_return); 651 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_null_pointer_exception), false); 652 } 653 break; 654 655 case new_instance_id: 656 case fast_new_instance_id: 657 case fast_new_instance_init_check_id: 658 { 659 Register klass = x13; // Incoming 660 Register obj = x10; // Result 661 662 if (id == new_instance_id) { 663 __ set_info("new_instance", dont_gc_arguments); 664 } else if (id == fast_new_instance_id) { 665 __ set_info("fast new_instance", dont_gc_arguments); 666 } else { 667 assert(id == fast_new_instance_init_check_id, "bad StubID"); 668 __ set_info("fast new_instance init check", dont_gc_arguments); 669 } 670 671 // If TLAB is disabled, see if there is support for inlining contiguous 672 // allocations. 673 // Otherwise, just go to the slow path. 674 if ((id == fast_new_instance_id || id == fast_new_instance_init_check_id) && 675 !UseTLAB && Universe::heap()->supports_inline_contig_alloc()) { 676 Label slow_path; 677 Register obj_size = x12; 678 Register tmp1 = x9; 679 Register tmp2 = x14; 680 assert_different_registers(klass, obj, obj_size, tmp1, tmp2); 681 682 const int sp_offset = 2; 683 const int x9_offset = 1; 684 const int zr_offset = 0; 685 __ addi(sp, sp, -(sp_offset * wordSize)); 686 __ sd(x9, Address(sp, x9_offset * wordSize)); 687 __ sd(zr, Address(sp, zr_offset * wordSize)); 688 689 if (id == fast_new_instance_init_check_id) { 690 // make sure the klass is initialized 691 __ lbu(t0, Address(klass, InstanceKlass::init_state_offset())); 692 __ mv(t1, InstanceKlass::fully_initialized); 693 __ bne(t0, t1, slow_path); 694 } 695 696 #ifdef ASSERT 697 // assert object can be fast path allocated 698 { 699 Label ok, not_ok; 700 __ lw(obj_size, Address(klass, Klass::layout_helper_offset())); 701 // make sure it's an instance. For instances, layout helper is a positive number. 702 // For arrays, layout helper is a negative number 703 __ blez(obj_size, not_ok); 704 __ andi(t0, obj_size, Klass::_lh_instance_slow_path_bit); 705 __ beqz(t0, ok); 706 __ bind(not_ok); 707 __ stop("assert(can be fast path allocated)"); 708 __ should_not_reach_here(); 709 __ bind(ok); 710 } 711 #endif // ASSERT 712 713 // get the instance size 714 __ lwu(obj_size, Address(klass, Klass::layout_helper_offset())); 715 716 __ eden_allocate(obj, obj_size, 0, tmp1, slow_path); 717 718 __ initialize_object(obj, klass, obj_size, 0, tmp1, tmp2, /* is_tlab_allocated */ false); 719 __ verify_oop(obj); 720 __ ld(x9, Address(sp, x9_offset * wordSize)); 721 __ ld(zr, Address(sp, zr_offset * wordSize)); 722 __ addi(sp, sp, sp_offset * wordSize); 723 __ ret(); 724 725 __ bind(slow_path); 726 __ ld(x9, Address(sp, x9_offset * wordSize)); 727 __ ld(zr, Address(sp, zr_offset * wordSize)); 728 __ addi(sp, sp, sp_offset * wordSize); 729 } 730 731 __ enter(); 732 OopMap* map = save_live_registers(sasm); 733 assert_cond(map != NULL); 734 int call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_instance), klass); 735 oop_maps = new OopMapSet(); 736 assert_cond(oop_maps != NULL); 737 oop_maps->add_gc_map(call_offset, map); 738 restore_live_registers_except_r10(sasm); 739 __ verify_oop(obj); 740 __ leave(); 741 __ ret(); 742 743 // x10: new instance 744 } 745 746 break; 747 748 case counter_overflow_id: 749 { 750 Register bci = x10; 751 Register method = x11; 752 __ enter(); 753 OopMap* map = save_live_registers(sasm); 754 assert_cond(map != NULL); 755 756 const int bci_off = 0; 757 const int method_off = 1; 758 // Retrieve bci 759 __ lw(bci, Address(fp, bci_off * BytesPerWord)); 760 // And a pointer to the Method* 761 __ ld(method, Address(fp, method_off * BytesPerWord)); 762 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, counter_overflow), bci, method); 763 oop_maps = new OopMapSet(); 764 assert_cond(oop_maps != NULL); 765 oop_maps->add_gc_map(call_offset, map); 766 restore_live_registers(sasm); 767 __ leave(); 768 __ ret(); 769 } 770 break; 771 772 case new_type_array_id: 773 case new_object_array_id: 774 { 775 Register length = x9; // Incoming 776 Register klass = x13; // Incoming 777 Register obj = x10; // Result 778 779 if (id == new_type_array_id) { 780 __ set_info("new_type_array", dont_gc_arguments); 781 } else { 782 __ set_info("new_object_array", dont_gc_arguments); 783 } 784 785 #ifdef ASSERT 786 // assert object type is really an array of the proper kind 787 { 788 Label ok; 789 Register tmp = obj; 790 __ lwu(tmp, Address(klass, Klass::layout_helper_offset())); 791 __ sraiw(tmp, tmp, Klass::_lh_array_tag_shift); 792 int tag = ((id == new_type_array_id) ? Klass::_lh_array_tag_type_value : Klass::_lh_array_tag_obj_value); 793 __ mv(t0, tag); 794 __ beq(t0, tmp, ok); 795 __ stop("assert(is an array klass)"); 796 __ should_not_reach_here(); 797 __ bind(ok); 798 } 799 #endif // ASSERT 800 801 // If TLAB is disabled, see if there is support for inlining contiguous 802 // allocations. 803 // Otherwise, just go to the slow path. 804 if (!UseTLAB && Universe::heap()->supports_inline_contig_alloc()) { 805 Register arr_size = x14; 806 Register tmp1 = x12; 807 Register tmp2 = x15; 808 Label slow_path; 809 assert_different_registers(length, klass, obj, arr_size, tmp1, tmp2); 810 811 // check that array length is small enough for fast path. 812 __ mv(t0, C1_MacroAssembler::max_array_allocation_length); 813 __ bgtu(length, t0, slow_path); 814 815 // get the allocation size: round_up(hdr + length << (layout_helper & 0x1F)) 816 __ lwu(tmp1, Address(klass, Klass::layout_helper_offset())); 817 __ andi(t0, tmp1, 0x1f); 818 __ sll(arr_size, length, t0); 819 int lh_header_size_width = exact_log2(Klass::_lh_header_size_mask + 1); 820 int lh_header_size_msb = Klass::_lh_header_size_shift + lh_header_size_width; 821 __ slli(tmp1, tmp1, XLEN - lh_header_size_msb); 822 __ srli(tmp1, tmp1, XLEN - lh_header_size_width); 823 __ add(arr_size, arr_size, tmp1); 824 __ addi(arr_size, arr_size, MinObjAlignmentInBytesMask); // align up 825 __ andi(arr_size, arr_size, ~(uint)MinObjAlignmentInBytesMask); 826 827 __ eden_allocate(obj, arr_size, 0, tmp1, slow_path); // preserves arr_size 828 829 __ initialize_header(obj, klass, length, tmp1, tmp2); 830 __ lbu(tmp1, Address(klass, 831 in_bytes(Klass::layout_helper_offset()) + 832 (Klass::_lh_header_size_shift / BitsPerByte))); 833 assert(Klass::_lh_header_size_shift % BitsPerByte == 0, "bytewise"); 834 assert(Klass::_lh_header_size_mask <= 0xFF, "bytewise"); 835 __ andi(tmp1, tmp1, Klass::_lh_header_size_mask); 836 __ sub(arr_size, arr_size, tmp1); // body length 837 __ add(tmp1, tmp1, obj); // body start 838 __ initialize_body(tmp1, arr_size, 0, tmp2); 839 __ membar(MacroAssembler::StoreStore); 840 __ verify_oop(obj); 841 842 __ ret(); 843 844 __ bind(slow_path); 845 } 846 847 __ enter(); 848 OopMap* map = save_live_registers(sasm); 849 assert_cond(map != NULL); 850 int call_offset = 0; 851 if (id == new_type_array_id) { 852 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_type_array), klass, length); 853 } else { 854 call_offset = __ call_RT(obj, noreg, CAST_FROM_FN_PTR(address, new_object_array), klass, length); 855 } 856 857 oop_maps = new OopMapSet(); 858 assert_cond(oop_maps != NULL); 859 oop_maps->add_gc_map(call_offset, map); 860 restore_live_registers_except_r10(sasm); 861 862 __ verify_oop(obj); 863 __ leave(); 864 __ ret(); 865 866 // x10: new array 867 } 868 break; 869 870 case new_multi_array_id: 871 { 872 StubFrame f(sasm, "new_multi_array", dont_gc_arguments); 873 // x10: klass 874 // x9: rank 875 // x12: address of 1st dimension 876 OopMap* map = save_live_registers(sasm); 877 assert_cond(map != NULL); 878 __ mv(c_rarg1, x10); 879 __ mv(c_rarg3, x12); 880 __ mv(c_rarg2, x9); 881 int call_offset = __ call_RT(x10, noreg, CAST_FROM_FN_PTR(address, new_multi_array), x11, x12, x13); 882 883 oop_maps = new OopMapSet(); 884 assert_cond(oop_maps != NULL); 885 oop_maps->add_gc_map(call_offset, map); 886 restore_live_registers_except_r10(sasm); 887 888 // x10: new multi array 889 __ verify_oop(x10); 890 } 891 break; 892 893 case register_finalizer_id: 894 { 895 __ set_info("register_finalizer", dont_gc_arguments); 896 897 // This is called via call_runtime so the arguments 898 // will be place in C abi locations 899 __ verify_oop(c_rarg0); 900 901 // load the klass and check the has finalizer flag 902 Label register_finalizer; 903 Register t = x15; 904 __ load_klass(t, x10); 905 __ lwu(t, Address(t, Klass::access_flags_offset())); 906 __ andi(t0, t, JVM_ACC_HAS_FINALIZER); 907 __ bnez(t0, register_finalizer); 908 __ ret(); 909 910 __ bind(register_finalizer); 911 __ enter(); 912 OopMap* oop_map = save_live_registers(sasm); 913 assert_cond(oop_map != NULL); 914 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, SharedRuntime::register_finalizer), x10); 915 oop_maps = new OopMapSet(); 916 assert_cond(oop_maps != NULL); 917 oop_maps->add_gc_map(call_offset, oop_map); 918 919 // Now restore all the live registers 920 restore_live_registers(sasm); 921 922 __ leave(); 923 __ ret(); 924 } 925 break; 926 927 case throw_class_cast_exception_id: 928 { 929 StubFrame f(sasm, "throw_class_cast_exception", dont_gc_arguments, does_not_return); 930 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_class_cast_exception), true); 931 } 932 break; 933 934 case throw_incompatible_class_change_error_id: 935 { 936 StubFrame f(sasm, "throw_incompatible_class_cast_exception", dont_gc_arguments, does_not_return); 937 oop_maps = generate_exception_throw(sasm, 938 CAST_FROM_FN_PTR(address, throw_incompatible_class_change_error), false); 939 } 940 break; 941 942 case slow_subtype_check_id: 943 { 944 // Typical calling sequence: 945 // push klass_RInfo (object klass or other subclass) 946 // push sup_k_RInfo (array element klass or other superclass) 947 // jump to slow_subtype_check 948 // Note that the subclass is pushed first, and is therefore deepest. 949 enum layout { 950 x10_off, x10_off_hi, 951 x12_off, x12_off_hi, 952 x14_off, x14_off_hi, 953 x15_off, x15_off_hi, 954 sup_k_off, sup_k_off_hi, 955 klass_off, klass_off_hi, 956 framesize, 957 result_off = sup_k_off 958 }; 959 960 __ set_info("slow_subtype_check", dont_gc_arguments); 961 __ push_reg(RegSet::of(x10, x12, x14, x15), sp); 962 963 __ ld(x14, Address(sp, (klass_off) * VMRegImpl::stack_slot_size)); // sub klass 964 __ ld(x10, Address(sp, (sup_k_off) * VMRegImpl::stack_slot_size)); // super klass 965 966 Label miss; 967 __ check_klass_subtype_slow_path(x14, x10, x12, x15, NULL, &miss); 968 969 // fallthrough on success: 970 __ li(t0, 1); 971 __ sd(t0, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result 972 __ pop_reg(RegSet::of(x10, x12, x14, x15), sp); 973 __ ret(); 974 975 __ bind(miss); 976 __ sd(zr, Address(sp, (result_off) * VMRegImpl::stack_slot_size)); // result 977 __ pop_reg(RegSet::of(x10, x12, x14, x15), sp); 978 __ ret(); 979 } 980 break; 981 982 case monitorenter_nofpu_id: 983 save_fpu_registers = false; 984 // fall through 985 case monitorenter_id: 986 { 987 StubFrame f(sasm, "monitorenter", dont_gc_arguments); 988 OopMap* map = save_live_registers(sasm, save_fpu_registers); 989 assert_cond(map != NULL); 990 991 // Called with store_parameter and not C abi 992 f.load_argument(1, x10); // x10: object 993 f.load_argument(0, x11); // x11: lock address 994 995 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorenter), x10, x11); 996 997 oop_maps = new OopMapSet(); 998 assert_cond(oop_maps != NULL); 999 oop_maps->add_gc_map(call_offset, map); 1000 restore_live_registers(sasm, save_fpu_registers); 1001 } 1002 break; 1003 1004 case monitorexit_nofpu_id: 1005 save_fpu_registers = false; 1006 // fall through 1007 case monitorexit_id: 1008 { 1009 StubFrame f(sasm, "monitorexit", dont_gc_arguments); 1010 OopMap* map = save_live_registers(sasm, save_fpu_registers); 1011 assert_cond(map != NULL); 1012 1013 // Called with store_parameter and not C abi 1014 f.load_argument(0, x10); // x10: lock address 1015 1016 // note: really a leaf routine but must setup last java sp 1017 // => use call_RT for now (speed can be improved by 1018 // doing last java sp setup manually) 1019 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, monitorexit), x10); 1020 1021 oop_maps = new OopMapSet(); 1022 assert_cond(oop_maps != NULL); 1023 oop_maps->add_gc_map(call_offset, map); 1024 restore_live_registers(sasm, save_fpu_registers); 1025 } 1026 break; 1027 1028 case deoptimize_id: 1029 { 1030 StubFrame f(sasm, "deoptimize", dont_gc_arguments, does_not_return); 1031 OopMap* oop_map = save_live_registers(sasm); 1032 assert_cond(oop_map != NULL); 1033 f.load_argument(0, c_rarg1); 1034 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, deoptimize), c_rarg1); 1035 1036 oop_maps = new OopMapSet(); 1037 assert_cond(oop_maps != NULL); 1038 oop_maps->add_gc_map(call_offset, oop_map); 1039 restore_live_registers(sasm); 1040 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); 1041 assert(deopt_blob != NULL, "deoptimization blob must have been created"); 1042 __ leave(); 1043 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); 1044 } 1045 break; 1046 1047 case throw_range_check_failed_id: 1048 { 1049 StubFrame f(sasm, "range_check_failed", dont_gc_arguments, does_not_return); 1050 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_range_check_exception), true); 1051 } 1052 break; 1053 1054 case unwind_exception_id: 1055 { 1056 __ set_info("unwind_exception", dont_gc_arguments); 1057 // note: no stubframe since we are about to leave the current 1058 // activation and we are calling a leaf VM function only. 1059 generate_unwind_exception(sasm); 1060 } 1061 break; 1062 1063 case access_field_patching_id: 1064 { 1065 StubFrame f(sasm, "access_field_patching", dont_gc_arguments, does_not_return); 1066 // we should set up register map 1067 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, access_field_patching)); 1068 } 1069 break; 1070 1071 case load_klass_patching_id: 1072 { 1073 StubFrame f(sasm, "load_klass_patching", dont_gc_arguments, does_not_return); 1074 // we should set up register map 1075 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_klass_patching)); 1076 } 1077 break; 1078 1079 case load_mirror_patching_id: 1080 { 1081 StubFrame f(sasm, "load_mirror_patching", dont_gc_arguments, does_not_return); 1082 // we should set up register map 1083 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_mirror_patching)); 1084 } 1085 break; 1086 1087 case load_appendix_patching_id: 1088 { 1089 StubFrame f(sasm, "load_appendix_patching", dont_gc_arguments, does_not_return); 1090 // we should set up register map 1091 oop_maps = generate_patching(sasm, CAST_FROM_FN_PTR(address, move_appendix_patching)); 1092 } 1093 break; 1094 1095 case handle_exception_nofpu_id: 1096 case handle_exception_id: 1097 { 1098 StubFrame f(sasm, "handle_exception", dont_gc_arguments); 1099 oop_maps = generate_handle_exception(id, sasm); 1100 } 1101 break; 1102 1103 case handle_exception_from_callee_id: 1104 { 1105 StubFrame f(sasm, "handle_exception_from_callee", dont_gc_arguments); 1106 oop_maps = generate_handle_exception(id, sasm); 1107 } 1108 break; 1109 1110 case throw_index_exception_id: 1111 { 1112 StubFrame f(sasm, "index_range_check_failed", dont_gc_arguments, does_not_return); 1113 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_index_exception), true); 1114 } 1115 break; 1116 1117 case throw_array_store_exception_id: 1118 { 1119 StubFrame f(sasm, "throw_array_store_exception", dont_gc_arguments, does_not_return); 1120 // tos + 0: link 1121 // + 1: return address 1122 oop_maps = generate_exception_throw(sasm, CAST_FROM_FN_PTR(address, throw_array_store_exception), true); 1123 } 1124 break; 1125 1126 case predicate_failed_trap_id: 1127 { 1128 StubFrame f(sasm, "predicate_failed_trap", dont_gc_arguments, does_not_return); 1129 1130 OopMap* map = save_live_registers(sasm); 1131 assert_cond(map != NULL); 1132 1133 int call_offset = __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, predicate_failed_trap)); 1134 oop_maps = new OopMapSet(); 1135 assert_cond(oop_maps != NULL); 1136 oop_maps->add_gc_map(call_offset, map); 1137 restore_live_registers(sasm); 1138 __ leave(); 1139 DeoptimizationBlob* deopt_blob = SharedRuntime::deopt_blob(); 1140 assert(deopt_blob != NULL, "deoptimization blob must have been created"); 1141 1142 __ far_jump(RuntimeAddress(deopt_blob->unpack_with_reexecution())); 1143 } 1144 break; 1145 1146 case dtrace_object_alloc_id: 1147 { // c_rarg0: object 1148 StubFrame f(sasm, "dtrace_object_alloc", dont_gc_arguments); 1149 save_live_registers(sasm); 1150 1151 __ call_VM_leaf(CAST_FROM_FN_PTR(address, static_cast<int (*)(oopDesc*)>(SharedRuntime::dtrace_object_alloc)), c_rarg0); 1152 1153 restore_live_registers(sasm); 1154 } 1155 break; 1156 1157 default: 1158 { 1159 StubFrame f(sasm, "unimplemented entry", dont_gc_arguments, does_not_return); 1160 __ li(x10, (int) id); 1161 __ call_RT(noreg, noreg, CAST_FROM_FN_PTR(address, unimplemented_entry), x10); 1162 __ should_not_reach_here(); 1163 } 1164 break; 1165 } 1166 } 1167 return oop_maps; 1168 } 1169 1170 #undef __ 1171 1172 const char *Runtime1::pd_name_for_address(address entry) { Unimplemented(); return 0; }