1 /* 2 * Copyright (c) 2000, 2023, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. 4 * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 #include "asm/assembler.hpp" 29 #include "asm/macroAssembler.inline.hpp" 30 #include "c1/c1_CodeStubs.hpp" 31 #include "c1/c1_Compilation.hpp" 32 #include "c1/c1_LIRAssembler.hpp" 33 #include "c1/c1_MacroAssembler.hpp" 34 #include "c1/c1_Runtime1.hpp" 35 #include "c1/c1_ValueStack.hpp" 36 #include "ci/ciArrayKlass.hpp" 37 #include "ci/ciInstance.hpp" 38 #include "code/compiledIC.hpp" 39 #include "gc/shared/collectedHeap.hpp" 40 #include "nativeInst_riscv.hpp" 41 #include "oops/objArrayKlass.hpp" 42 #include "runtime/frame.inline.hpp" 43 #include "runtime/sharedRuntime.hpp" 44 #include "utilities/powerOfTwo.hpp" 45 #include "vmreg_riscv.inline.hpp" 46 47 #ifndef PRODUCT 48 #define COMMENT(x) do { __ block_comment(x); } while (0) 49 #else 50 #define COMMENT(x) 51 #endif 52 53 NEEDS_CLEANUP // remove this definitions ? 54 const Register IC_Klass = t1; // where the IC klass is cached 55 const Register SYNC_header = x10; // synchronization header 56 const Register SHIFT_count = x10; // where count for shift operations must be 57 58 #define __ _masm-> 59 60 static void select_different_registers(Register preserve, 61 Register extra, 62 Register &tmp1, 63 Register &tmp2) { 64 if (tmp1 == preserve) { 65 assert_different_registers(tmp1, tmp2, extra); 66 tmp1 = extra; 67 } else if (tmp2 == preserve) { 68 assert_different_registers(tmp1, tmp2, extra); 69 tmp2 = extra; 70 } 71 assert_different_registers(preserve, tmp1, tmp2); 72 } 73 74 static void select_different_registers(Register preserve, 75 Register extra, 76 Register &tmp1, 77 Register &tmp2, 78 Register &tmp3) { 79 if (tmp1 == preserve) { 80 assert_different_registers(tmp1, tmp2, tmp3, extra); 81 tmp1 = extra; 82 } else if (tmp2 == preserve) { 83 assert_different_registers(tmp1, tmp2, tmp3, extra); 84 tmp2 = extra; 85 } else if (tmp3 == preserve) { 86 assert_different_registers(tmp1, tmp2, tmp3, extra); 87 tmp3 = extra; 88 } 89 assert_different_registers(preserve, tmp1, tmp2, tmp3); 90 } 91 92 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; } 93 94 void LIR_Assembler::clinit_barrier(ciMethod* method) { 95 assert(VM_Version::supports_fast_class_init_checks(), "sanity"); 96 assert(!method->holder()->is_not_initialized(), "initialization should have been started"); 97 98 Label L_skip_barrier; 99 100 __ mov_metadata(t1, method->holder()->constant_encoding()); 101 __ clinit_barrier(t1, t0, &L_skip_barrier /* L_fast_path */); 102 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); 103 __ bind(L_skip_barrier); 104 } 105 106 LIR_Opr LIR_Assembler::receiverOpr() { 107 return FrameMap::receiver_opr; 108 } 109 110 LIR_Opr LIR_Assembler::osrBufferPointer() { 111 return FrameMap::as_pointer_opr(receiverOpr()->as_register()); 112 } 113 114 void LIR_Assembler::breakpoint() { Unimplemented(); } 115 116 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); } 117 118 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); } 119 120 static jlong as_long(LIR_Opr data) { 121 jlong result; 122 switch (data->type()) { 123 case T_INT: 124 result = (data->as_jint()); 125 break; 126 case T_LONG: 127 result = (data->as_jlong()); 128 break; 129 default: 130 ShouldNotReachHere(); 131 result = 0; // unreachable 132 } 133 return result; 134 } 135 136 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) { 137 if (addr->base()->is_illegal()) { 138 assert(addr->index()->is_illegal(), "must be illegal too"); 139 __ movptr(tmp, addr->disp()); 140 return Address(tmp, 0); 141 } 142 143 Register base = addr->base()->as_pointer_register(); 144 LIR_Opr index_opr = addr->index(); 145 146 if (index_opr->is_illegal()) { 147 return Address(base, addr->disp()); 148 } 149 150 int scale = addr->scale(); 151 if (index_opr->is_cpu_register()) { 152 Register index; 153 if (index_opr->is_single_cpu()) { 154 index = index_opr->as_register(); 155 } else { 156 index = index_opr->as_register_lo(); 157 } 158 if (scale != 0) { 159 __ shadd(tmp, index, base, tmp, scale); 160 } else { 161 __ add(tmp, base, index); 162 } 163 return Address(tmp, addr->disp()); 164 } else if (index_opr->is_constant()) { 165 intptr_t addr_offset = (((intptr_t)index_opr->as_constant_ptr()->as_jint()) << scale) + addr->disp(); 166 return Address(base, addr_offset); 167 } 168 169 Unimplemented(); 170 return Address(); 171 } 172 173 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { 174 ShouldNotReachHere(); 175 return Address(); 176 } 177 178 Address LIR_Assembler::as_Address(LIR_Address* addr) { 179 return as_Address(addr, t0); 180 } 181 182 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { 183 return as_Address(addr); 184 } 185 186 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is 187 // not encodable as a base + (immediate) offset, generate an explicit address 188 // calculation to hold the address in t0. 189 Address LIR_Assembler::stack_slot_address(int index, uint size, int adjust) { 190 precond(size == 4 || size == 8); 191 Address addr = frame_map()->address_for_slot(index, adjust); 192 precond(addr.getMode() == Address::base_plus_offset); 193 precond(addr.base() == sp); 194 precond(addr.offset() > 0); 195 uint mask = size - 1; 196 assert((addr.offset() & mask) == 0, "scaled offsets only"); 197 198 return addr; 199 } 200 201 void LIR_Assembler::osr_entry() { 202 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); 203 BlockBegin* osr_entry = compilation()->hir()->osr_entry(); 204 guarantee(osr_entry != nullptr, "null osr_entry!"); 205 ValueStack* entry_state = osr_entry->state(); 206 int number_of_locks = entry_state->locks_size(); 207 208 // we jump here if osr happens with the interpreter 209 // state set up to continue at the beginning of the 210 // loop that triggered osr - in particular, we have 211 // the following registers setup: 212 // 213 // x12: osr buffer 214 // 215 216 //build frame 217 ciMethod* m = compilation()->method(); 218 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); 219 220 // OSR buffer is 221 // 222 // locals[nlocals-1..0] 223 // monitors[0..number_of_locks] 224 // 225 // locals is a direct copy of the interpreter frame so in the osr buffer 226 // so first slot in the local array is the last local from the interpreter 227 // and last slot is local[0] (receiver) from the interpreter 228 // 229 // Similarly with locks. The first lock slot in the osr buffer is the nth lock 230 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock 231 // in the interpreter frame (the method lock if a sync method) 232 233 // Initialize monitors in the compiled activation. 234 // x12: pointer to osr buffer 235 // All other registers are dead at this point and the locals will be 236 // copied into place by code emitted in the IR. 237 238 Register OSR_buf = osrBufferPointer()->as_pointer_register(); 239 { 240 assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); 241 int monitor_offset = BytesPerWord * method()->max_locals() + 242 (2 * BytesPerWord) * (number_of_locks - 1); 243 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in 244 // the OSR buffer using 2 word entries: first the lock and then 245 // the oop. 246 for (int i = 0; i < number_of_locks; i++) { 247 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord); 248 #ifdef ASSERT 249 // verify the interpreter's monitor has a non-null object 250 { 251 Label L; 252 __ ld(t0, Address(OSR_buf, slot_offset + 1 * BytesPerWord)); 253 __ bnez(t0, L); 254 __ stop("locked object is null"); 255 __ bind(L); 256 } 257 #endif // ASSERT 258 __ ld(x9, Address(OSR_buf, slot_offset + 0)); 259 __ sd(x9, frame_map()->address_for_monitor_lock(i)); 260 __ ld(x9, Address(OSR_buf, slot_offset + 1 * BytesPerWord)); 261 __ sd(x9, frame_map()->address_for_monitor_object(i)); 262 } 263 } 264 } 265 266 // inline cache check; done before the frame is built. 267 int LIR_Assembler::check_icache() { 268 Register receiver = FrameMap::receiver_opr->as_register(); 269 Register ic_klass = IC_Klass; 270 int start_offset = __ offset(); 271 Label dont; 272 __ inline_cache_check(receiver, ic_klass, dont); 273 274 // if icache check fails, then jump to runtime routine 275 // Note: RECEIVER must still contain the receiver! 276 __ far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 277 278 // We align the verified entry point unless the method body 279 // (including its inline cache check) will fit in a single 64-byte 280 // icache line. 281 if (!method()->is_accessor() || __ offset() - start_offset > 4 * 4) { 282 // force alignment after the cache check. 283 __ align(CodeEntryAlignment); 284 } 285 286 __ bind(dont); 287 return start_offset; 288 } 289 290 void LIR_Assembler::jobject2reg(jobject o, Register reg) { 291 if (o == nullptr) { 292 __ mv(reg, zr); 293 } else { 294 __ movoop(reg, o); 295 } 296 } 297 298 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) { 299 deoptimize_trap(info); 300 } 301 302 // This specifies the rsp decrement needed to build the frame 303 int LIR_Assembler::initial_frame_size_in_bytes() const { 304 // if rounding, must let FrameMap know! 305 306 return in_bytes(frame_map()->framesize_in_bytes()); 307 } 308 309 int LIR_Assembler::emit_exception_handler() { 310 // generate code for exception handler 311 address handler_base = __ start_a_stub(exception_handler_size()); 312 if (handler_base == nullptr) { 313 // not enough space left for the handler 314 bailout("exception handler overflow"); 315 return -1; 316 } 317 318 int offset = code_offset(); 319 320 // the exception oop and pc are in x10, and x13 321 // no other registers need to be preserved, so invalidate them 322 __ invalidate_registers(false, true, true, false, true, true); 323 324 // check that there is really an exception 325 __ verify_not_null_oop(x10); 326 327 // search an exception handler (x10: exception oop, x13: throwing pc) 328 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::handle_exception_from_callee_id))); 329 __ should_not_reach_here(); 330 guarantee(code_offset() - offset <= exception_handler_size(), "overflow"); 331 __ end_a_stub(); 332 333 return offset; 334 } 335 336 // Emit the code to remove the frame from the stack in the exception 337 // unwind path. 338 int LIR_Assembler::emit_unwind_handler() { 339 #ifndef PRODUCT 340 if (CommentedAssembly) { 341 _masm->block_comment("Unwind handler"); 342 } 343 #endif // PRODUCT 344 345 int offset = code_offset(); 346 347 // Fetch the exception from TLS and clear out exception related thread state 348 __ ld(x10, Address(xthread, JavaThread::exception_oop_offset())); 349 __ sd(zr, Address(xthread, JavaThread::exception_oop_offset())); 350 __ sd(zr, Address(xthread, JavaThread::exception_pc_offset())); 351 352 __ bind(_unwind_handler_entry); 353 __ verify_not_null_oop(x10); 354 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 355 __ mv(x9, x10); // Preserve the exception 356 } 357 358 // Perform needed unlocking 359 MonitorExitStub* stub = nullptr; 360 if (method()->is_synchronized()) { 361 monitor_address(0, FrameMap::r10_opr); 362 stub = new MonitorExitStub(FrameMap::r10_opr, true, 0); 363 if (LockingMode == LM_MONITOR) { 364 __ j(*stub->entry()); 365 } else { 366 __ unlock_object(x15, x14, x10, *stub->entry()); 367 } 368 __ bind(*stub->continuation()); 369 } 370 371 if (compilation()->env()->dtrace_method_probes()) { 372 __ mv(c_rarg0, xthread); 373 __ mov_metadata(c_rarg1, method()->constant_encoding()); 374 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1); 375 } 376 377 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 378 __ mv(x10, x9); // Restore the exception 379 } 380 381 // remove the activation and dispatch to the unwind handler 382 __ block_comment("remove_frame and dispatch to the unwind handler"); 383 __ remove_frame(initial_frame_size_in_bytes()); 384 __ far_jump(RuntimeAddress(Runtime1::entry_for(Runtime1::unwind_exception_id))); 385 386 // Emit the slow path assembly 387 if (stub != nullptr) { 388 stub->emit_code(this); 389 } 390 391 return offset; 392 } 393 394 int LIR_Assembler::emit_deopt_handler() { 395 // generate code for exception handler 396 address handler_base = __ start_a_stub(deopt_handler_size()); 397 if (handler_base == nullptr) { 398 // not enough space left for the handler 399 bailout("deopt handler overflow"); 400 return -1; 401 } 402 403 int offset = code_offset(); 404 405 __ auipc(ra, 0); 406 __ far_jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); 407 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); 408 __ end_a_stub(); 409 410 return offset; 411 } 412 413 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { 414 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10"); 415 416 // Pop the stack before the safepoint code 417 __ remove_frame(initial_frame_size_in_bytes()); 418 419 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) { 420 __ reserved_stack_check(); 421 } 422 423 code_stub->set_safepoint_offset(__ offset()); 424 __ relocate(relocInfo::poll_return_type); 425 __ safepoint_poll(*code_stub->entry(), true /* at_return */, false /* acquire */, true /* in_nmethod */); 426 __ ret(); 427 } 428 429 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { 430 guarantee(info != nullptr, "Shouldn't be null"); 431 __ get_polling_page(t0, relocInfo::poll_type); 432 add_debug_info_for_branch(info); // This isn't just debug info: 433 // it's the oop map 434 __ read_polling_page(t0, 0, relocInfo::poll_type); 435 return __ offset(); 436 } 437 438 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { 439 __ mv(to_reg, from_reg); 440 } 441 442 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); } 443 444 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 445 assert(src->is_constant(), "should not call otherwise"); 446 assert(dest->is_register(), "should not call otherwise"); 447 LIR_Const* c = src->as_constant_ptr(); 448 address const_addr = nullptr; 449 450 switch (c->type()) { 451 case T_INT: 452 assert(patch_code == lir_patch_none, "no patching handled here"); 453 __ mv(dest->as_register(), c->as_jint()); 454 break; 455 456 case T_ADDRESS: 457 assert(patch_code == lir_patch_none, "no patching handled here"); 458 __ mv(dest->as_register(), c->as_jint()); 459 break; 460 461 case T_LONG: 462 assert(patch_code == lir_patch_none, "no patching handled here"); 463 __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong()); 464 break; 465 466 case T_OBJECT: 467 case T_ARRAY: 468 if (patch_code == lir_patch_none) { 469 jobject2reg(c->as_jobject(), dest->as_register()); 470 } else { 471 jobject2reg_with_patching(dest->as_register(), info); 472 } 473 break; 474 475 case T_METADATA: 476 if (patch_code != lir_patch_none) { 477 klass2reg_with_patching(dest->as_register(), info); 478 } else { 479 __ mov_metadata(dest->as_register(), c->as_metadata()); 480 } 481 break; 482 483 case T_FLOAT: 484 const_addr = float_constant(c->as_jfloat()); 485 assert(const_addr != nullptr, "must create float constant in the constant table"); 486 __ flw(dest->as_float_reg(), InternalAddress(const_addr)); 487 break; 488 489 case T_DOUBLE: 490 const_addr = double_constant(c->as_jdouble()); 491 assert(const_addr != nullptr, "must create double constant in the constant table"); 492 __ fld(dest->as_double_reg(), InternalAddress(const_addr)); 493 break; 494 495 default: 496 ShouldNotReachHere(); 497 } 498 } 499 500 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { 501 assert(src->is_constant(), "should not call otherwise"); 502 assert(dest->is_stack(), "should not call otherwise"); 503 LIR_Const* c = src->as_constant_ptr(); 504 switch (c->type()) { 505 case T_OBJECT: 506 if (c->as_jobject() == nullptr) { 507 __ sd(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 508 } else { 509 const2reg(src, FrameMap::t1_opr, lir_patch_none, nullptr); 510 reg2stack(FrameMap::t1_opr, dest, c->type(), false); 511 } 512 break; 513 case T_ADDRESS: // fall through 514 const2reg(src, FrameMap::t1_opr, lir_patch_none, nullptr); 515 reg2stack(FrameMap::t1_opr, dest, c->type(), false); 516 case T_INT: // fall through 517 case T_FLOAT: 518 if (c->as_jint_bits() == 0) { 519 __ sw(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 520 } else { 521 __ mv(t1, c->as_jint_bits()); 522 __ sw(t1, frame_map()->address_for_slot(dest->single_stack_ix())); 523 } 524 break; 525 case T_LONG: // fall through 526 case T_DOUBLE: 527 if (c->as_jlong_bits() == 0) { 528 __ sd(zr, frame_map()->address_for_slot(dest->double_stack_ix(), 529 lo_word_offset_in_bytes)); 530 } else { 531 __ mv(t1, (intptr_t)c->as_jlong_bits()); 532 __ sd(t1, frame_map()->address_for_slot(dest->double_stack_ix(), 533 lo_word_offset_in_bytes)); 534 } 535 break; 536 default: 537 ShouldNotReachHere(); 538 } 539 } 540 541 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) { 542 assert(src->is_constant(), "should not call otherwise"); 543 assert(dest->is_address(), "should not call otherwise"); 544 LIR_Const* c = src->as_constant_ptr(); 545 LIR_Address* to_addr = dest->as_address_ptr(); 546 void (MacroAssembler::* insn)(Register Rt, const Address &adr, Register temp); 547 switch (type) { 548 case T_ADDRESS: 549 assert(c->as_jint() == 0, "should be"); 550 insn = &MacroAssembler::sd; break; 551 case T_LONG: 552 assert(c->as_jlong() == 0, "should be"); 553 insn = &MacroAssembler::sd; break; 554 case T_DOUBLE: 555 assert(c->as_jdouble() == 0.0, "should be"); 556 insn = &MacroAssembler::sd; break; 557 case T_INT: 558 assert(c->as_jint() == 0, "should be"); 559 insn = &MacroAssembler::sw; break; 560 case T_FLOAT: 561 assert(c->as_jfloat() == 0.0f, "should be"); 562 insn = &MacroAssembler::sw; break; 563 case T_OBJECT: // fall through 564 case T_ARRAY: 565 assert(c->as_jobject() == 0, "should be"); 566 if (UseCompressedOops && !wide) { 567 insn = &MacroAssembler::sw; 568 } else { 569 insn = &MacroAssembler::sd; 570 } 571 break; 572 case T_CHAR: // fall through 573 case T_SHORT: 574 assert(c->as_jint() == 0, "should be"); 575 insn = &MacroAssembler::sh; 576 break; 577 case T_BOOLEAN: // fall through 578 case T_BYTE: 579 assert(c->as_jint() == 0, "should be"); 580 insn = &MacroAssembler::sb; break; 581 default: 582 ShouldNotReachHere(); 583 insn = &MacroAssembler::sd; // unreachable 584 } 585 if (info != nullptr) { 586 add_debug_info_for_null_check_here(info); 587 } 588 (_masm->*insn)(zr, as_Address(to_addr), t0); 589 } 590 591 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { 592 assert(src->is_register(), "should not call otherwise"); 593 assert(dest->is_register(), "should not call otherwise"); 594 595 // move between cpu-registers 596 if (dest->is_single_cpu()) { 597 if (src->type() == T_LONG) { 598 // Can do LONG -> OBJECT 599 move_regs(src->as_register_lo(), dest->as_register()); 600 return; 601 } 602 assert(src->is_single_cpu(), "must match"); 603 if (src->type() == T_OBJECT) { 604 __ verify_oop(src->as_register()); 605 } 606 move_regs(src->as_register(), dest->as_register()); 607 } else if (dest->is_double_cpu()) { 608 if (is_reference_type(src->type())) { 609 __ verify_oop(src->as_register()); 610 move_regs(src->as_register(), dest->as_register_lo()); 611 return; 612 } 613 assert(src->is_double_cpu(), "must match"); 614 Register f_lo = src->as_register_lo(); 615 Register f_hi = src->as_register_hi(); 616 Register t_lo = dest->as_register_lo(); 617 Register t_hi = dest->as_register_hi(); 618 assert(f_hi == f_lo, "must be same"); 619 assert(t_hi == t_lo, "must be same"); 620 move_regs(f_lo, t_lo); 621 } else if (dest->is_single_fpu()) { 622 assert(src->is_single_fpu(), "expect single fpu"); 623 __ fmv_s(dest->as_float_reg(), src->as_float_reg()); 624 } else if (dest->is_double_fpu()) { 625 assert(src->is_double_fpu(), "expect double fpu"); 626 __ fmv_d(dest->as_double_reg(), src->as_double_reg()); 627 } else { 628 ShouldNotReachHere(); 629 } 630 } 631 632 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type, bool pop_fpu_stack) { 633 precond(src->is_register() && dest->is_stack()); 634 635 uint const c_sz32 = sizeof(uint32_t); 636 uint const c_sz64 = sizeof(uint64_t); 637 638 assert(src->is_register(), "should not call otherwise"); 639 assert(dest->is_stack(), "should not call otherwise"); 640 if (src->is_single_cpu()) { 641 int index = dest->single_stack_ix(); 642 if (is_reference_type(type)) { 643 __ sd(src->as_register(), stack_slot_address(index, c_sz64)); 644 __ verify_oop(src->as_register()); 645 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) { 646 __ sd(src->as_register(), stack_slot_address(index, c_sz64)); 647 } else { 648 __ sw(src->as_register(), stack_slot_address(index, c_sz32)); 649 } 650 } else if (src->is_double_cpu()) { 651 int index = dest->double_stack_ix(); 652 Address dest_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes); 653 __ sd(src->as_register_lo(), dest_addr_LO); 654 } else if (src->is_single_fpu()) { 655 int index = dest->single_stack_ix(); 656 __ fsw(src->as_float_reg(), stack_slot_address(index, c_sz32)); 657 } else if (src->is_double_fpu()) { 658 int index = dest->double_stack_ix(); 659 __ fsd(src->as_double_reg(), stack_slot_address(index, c_sz64)); 660 } else { 661 ShouldNotReachHere(); 662 } 663 } 664 665 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool pop_fpu_stack, bool wide) { 666 LIR_Address* to_addr = dest->as_address_ptr(); 667 // t0 was used as tmp reg in as_Address, so we use t1 as compressed_src 668 Register compressed_src = t1; 669 670 if (patch_code != lir_patch_none) { 671 deoptimize_trap(info); 672 return; 673 } 674 675 if (is_reference_type(type)) { 676 __ verify_oop(src->as_register()); 677 678 if (UseCompressedOops && !wide) { 679 __ encode_heap_oop(compressed_src, src->as_register()); 680 } else { 681 compressed_src = src->as_register(); 682 } 683 } 684 685 int null_check_here = code_offset(); 686 687 switch (type) { 688 case T_FLOAT: 689 __ fsw(src->as_float_reg(), as_Address(to_addr)); 690 break; 691 692 case T_DOUBLE: 693 __ fsd(src->as_double_reg(), as_Address(to_addr)); 694 break; 695 696 case T_ARRAY: // fall through 697 case T_OBJECT: 698 if (UseCompressedOops && !wide) { 699 __ sw(compressed_src, as_Address(to_addr)); 700 } else { 701 __ sd(compressed_src, as_Address(to_addr)); 702 } 703 break; 704 case T_METADATA: 705 // We get here to store a method pointer to the stack to pass to 706 // a dtrace runtime call. This can't work on 64 bit with 707 // compressed klass ptrs: T_METADATA can be compressed klass 708 // ptr or a 64 bit method pointer. 709 ShouldNotReachHere(); 710 __ sd(src->as_register(), as_Address(to_addr)); 711 break; 712 case T_ADDRESS: 713 __ sd(src->as_register(), as_Address(to_addr)); 714 break; 715 case T_INT: 716 __ sw(src->as_register(), as_Address(to_addr)); 717 break; 718 case T_LONG: 719 __ sd(src->as_register_lo(), as_Address(to_addr)); 720 break; 721 case T_BYTE: // fall through 722 case T_BOOLEAN: 723 __ sb(src->as_register(), as_Address(to_addr)); 724 break; 725 case T_CHAR: // fall through 726 case T_SHORT: 727 __ sh(src->as_register(), as_Address(to_addr)); 728 break; 729 default: 730 ShouldNotReachHere(); 731 } 732 733 if (info != nullptr) { 734 add_debug_info_for_null_check(null_check_here, info); 735 } 736 } 737 738 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { 739 precond(src->is_stack() && dest->is_register()); 740 741 uint const c_sz32 = sizeof(uint32_t); 742 uint const c_sz64 = sizeof(uint64_t); 743 744 if (dest->is_single_cpu()) { 745 int index = src->single_stack_ix(); 746 if (type == T_INT) { 747 __ lw(dest->as_register(), stack_slot_address(index, c_sz32)); 748 } else if (is_reference_type(type)) { 749 __ ld(dest->as_register(), stack_slot_address(index, c_sz64)); 750 __ verify_oop(dest->as_register()); 751 } else if (type == T_METADATA || type == T_ADDRESS) { 752 __ ld(dest->as_register(), stack_slot_address(index, c_sz64)); 753 } else { 754 __ lwu(dest->as_register(), stack_slot_address(index, c_sz32)); 755 } 756 } else if (dest->is_double_cpu()) { 757 int index = src->double_stack_ix(); 758 Address src_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes); 759 __ ld(dest->as_register_lo(), src_addr_LO); 760 } else if (dest->is_single_fpu()) { 761 int index = src->single_stack_ix(); 762 __ flw(dest->as_float_reg(), stack_slot_address(index, c_sz32)); 763 } else if (dest->is_double_fpu()) { 764 int index = src->double_stack_ix(); 765 __ fld(dest->as_double_reg(), stack_slot_address(index, c_sz64)); 766 } else { 767 ShouldNotReachHere(); 768 } 769 } 770 771 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { 772 deoptimize_trap(info); 773 } 774 775 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 776 LIR_Opr temp; 777 if (type == T_LONG || type == T_DOUBLE) { 778 temp = FrameMap::t1_long_opr; 779 } else { 780 temp = FrameMap::t1_opr; 781 } 782 783 stack2reg(src, temp, src->type()); 784 reg2stack(temp, dest, dest->type(), false); 785 } 786 787 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) { 788 assert(src->is_address(), "should not call otherwise"); 789 assert(dest->is_register(), "should not call otherwise"); 790 791 LIR_Address* addr = src->as_address_ptr(); 792 LIR_Address* from_addr = src->as_address_ptr(); 793 794 if (addr->base()->type() == T_OBJECT) { 795 __ verify_oop(addr->base()->as_pointer_register()); 796 } 797 798 if (patch_code != lir_patch_none) { 799 deoptimize_trap(info); 800 return; 801 } 802 803 if (info != nullptr) { 804 add_debug_info_for_null_check_here(info); 805 } 806 807 int null_check_here = code_offset(); 808 switch (type) { 809 case T_FLOAT: 810 __ flw(dest->as_float_reg(), as_Address(from_addr)); 811 break; 812 case T_DOUBLE: 813 __ fld(dest->as_double_reg(), as_Address(from_addr)); 814 break; 815 case T_ARRAY: // fall through 816 case T_OBJECT: 817 if (UseCompressedOops && !wide) { 818 __ lwu(dest->as_register(), as_Address(from_addr)); 819 } else { 820 __ ld(dest->as_register(), as_Address(from_addr)); 821 } 822 break; 823 case T_METADATA: 824 // We get here to store a method pointer to the stack to pass to 825 // a dtrace runtime call. This can't work on 64 bit with 826 // compressed klass ptrs: T_METADATA can be a compressed klass 827 // ptr or a 64 bit method pointer. 828 ShouldNotReachHere(); 829 __ ld(dest->as_register(), as_Address(from_addr)); 830 break; 831 case T_ADDRESS: 832 __ ld(dest->as_register(), as_Address(from_addr)); 833 break; 834 case T_INT: 835 __ lw(dest->as_register(), as_Address(from_addr)); 836 break; 837 case T_LONG: 838 __ ld(dest->as_register_lo(), as_Address_lo(from_addr)); 839 break; 840 case T_BYTE: 841 __ lb(dest->as_register(), as_Address(from_addr)); 842 break; 843 case T_BOOLEAN: 844 __ lbu(dest->as_register(), as_Address(from_addr)); 845 break; 846 case T_CHAR: 847 __ lhu(dest->as_register(), as_Address(from_addr)); 848 break; 849 case T_SHORT: 850 __ lh(dest->as_register(), as_Address(from_addr)); 851 break; 852 default: 853 ShouldNotReachHere(); 854 } 855 856 if (is_reference_type(type)) { 857 if (UseCompressedOops && !wide) { 858 __ decode_heap_oop(dest->as_register()); 859 } 860 861 if (!(UseZGC && !ZGenerational)) { 862 // Load barrier has not yet been applied, so ZGC can't verify the oop here 863 __ verify_oop(dest->as_register()); 864 } 865 } 866 } 867 868 void LIR_Assembler::emit_op3(LIR_Op3* op) { 869 switch (op->code()) { 870 case lir_idiv: // fall through 871 case lir_irem: 872 arithmetic_idiv(op->code(), 873 op->in_opr1(), 874 op->in_opr2(), 875 op->in_opr3(), 876 op->result_opr(), 877 op->info()); 878 break; 879 case lir_fmad: 880 __ fmadd_d(op->result_opr()->as_double_reg(), 881 op->in_opr1()->as_double_reg(), 882 op->in_opr2()->as_double_reg(), 883 op->in_opr3()->as_double_reg()); 884 break; 885 case lir_fmaf: 886 __ fmadd_s(op->result_opr()->as_float_reg(), 887 op->in_opr1()->as_float_reg(), 888 op->in_opr2()->as_float_reg(), 889 op->in_opr3()->as_float_reg()); 890 break; 891 default: 892 ShouldNotReachHere(); 893 } 894 } 895 896 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type, 897 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) { 898 Label label; 899 900 emit_branch(condition, cmp_opr1, cmp_opr2, label, /* is_far */ false, 901 /* is_unordered */ (condition == lir_cond_greaterEqual || condition == lir_cond_greater) ? false : true); 902 903 Label done; 904 move_op(opr2, result, type, lir_patch_none, nullptr, 905 false, // pop_fpu_stack 906 false); // wide 907 __ j(done); 908 __ bind(label); 909 move_op(opr1, result, type, lir_patch_none, nullptr, 910 false, // pop_fpu_stack 911 false); // wide 912 __ bind(done); 913 } 914 915 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { 916 LIR_Condition condition = op->cond(); 917 if (condition == lir_cond_always) { 918 if (op->info() != nullptr) { 919 add_debug_info_for_branch(op->info()); 920 } 921 } else { 922 assert(op->in_opr1() != LIR_OprFact::illegalOpr && op->in_opr2() != LIR_OprFact::illegalOpr, "conditional branches must have legal operands"); 923 } 924 bool is_unordered = (op->ublock() == op->block()); 925 emit_branch(condition, op->in_opr1(), op->in_opr2(), *op->label(), /* is_far */ true, is_unordered); 926 } 927 928 void LIR_Assembler::emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label, 929 bool is_far, bool is_unordered) { 930 931 if (cmp_flag == lir_cond_always) { 932 __ j(label); 933 return; 934 } 935 936 if (cmp1->is_cpu_register()) { 937 Register reg1 = as_reg(cmp1); 938 if (cmp2->is_cpu_register()) { 939 Register reg2 = as_reg(cmp2); 940 __ c1_cmp_branch(cmp_flag, reg1, reg2, label, cmp1->type(), is_far); 941 } else if (cmp2->is_constant()) { 942 const2reg_helper(cmp2); 943 __ c1_cmp_branch(cmp_flag, reg1, t0, label, cmp2->type(), is_far); 944 } else { 945 ShouldNotReachHere(); 946 } 947 } else if (cmp1->is_single_fpu()) { 948 assert(cmp2->is_single_fpu(), "expect single float register"); 949 __ c1_float_cmp_branch(cmp_flag, cmp1->as_float_reg(), cmp2->as_float_reg(), label, is_far, is_unordered); 950 } else if (cmp1->is_double_fpu()) { 951 assert(cmp2->is_double_fpu(), "expect double float register"); 952 __ c1_float_cmp_branch(cmp_flag | C1_MacroAssembler::c1_double_branch_mask, 953 cmp1->as_double_reg(), cmp2->as_double_reg(), label, is_far, is_unordered); 954 } else { 955 ShouldNotReachHere(); 956 } 957 } 958 959 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { 960 LIR_Opr src = op->in_opr(); 961 LIR_Opr dest = op->result_opr(); 962 963 switch (op->bytecode()) { 964 case Bytecodes::_i2f: 965 __ fcvt_s_w(dest->as_float_reg(), src->as_register()); break; 966 case Bytecodes::_i2d: 967 __ fcvt_d_w(dest->as_double_reg(), src->as_register()); break; 968 case Bytecodes::_l2d: 969 __ fcvt_d_l(dest->as_double_reg(), src->as_register_lo()); break; 970 case Bytecodes::_l2f: 971 __ fcvt_s_l(dest->as_float_reg(), src->as_register_lo()); break; 972 case Bytecodes::_f2d: 973 __ fcvt_d_s(dest->as_double_reg(), src->as_float_reg()); break; 974 case Bytecodes::_d2f: 975 __ fcvt_s_d(dest->as_float_reg(), src->as_double_reg()); break; 976 case Bytecodes::_i2c: 977 __ zero_extend(dest->as_register(), src->as_register(), 16); break; 978 case Bytecodes::_i2l: 979 __ sign_extend(dest->as_register_lo(), src->as_register(), 32); break; 980 case Bytecodes::_i2s: 981 __ sign_extend(dest->as_register(), src->as_register(), 16); break; 982 case Bytecodes::_i2b: 983 __ sign_extend(dest->as_register(), src->as_register(), 8); break; 984 case Bytecodes::_l2i: 985 __ sign_extend(dest->as_register(), src->as_register_lo(), 32); break; 986 case Bytecodes::_d2l: 987 __ fcvt_l_d_safe(dest->as_register_lo(), src->as_double_reg()); break; 988 case Bytecodes::_f2i: 989 __ fcvt_w_s_safe(dest->as_register(), src->as_float_reg()); break; 990 case Bytecodes::_f2l: 991 __ fcvt_l_s_safe(dest->as_register_lo(), src->as_float_reg()); break; 992 case Bytecodes::_d2i: 993 __ fcvt_w_d_safe(dest->as_register(), src->as_double_reg()); break; 994 default: 995 ShouldNotReachHere(); 996 } 997 } 998 999 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { 1000 if (op->init_check()) { 1001 __ lbu(t0, Address(op->klass()->as_register(), 1002 InstanceKlass::init_state_offset())); 1003 __ mv(t1, (u1)InstanceKlass::fully_initialized); 1004 add_debug_info_for_null_check_here(op->stub()->info()); 1005 __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true); 1006 } 1007 1008 __ allocate_object(op->obj()->as_register(), 1009 op->tmp1()->as_register(), 1010 op->tmp2()->as_register(), 1011 op->header_size(), 1012 op->object_size(), 1013 op->klass()->as_register(), 1014 *op->stub()->entry()); 1015 1016 __ bind(*op->stub()->continuation()); 1017 } 1018 1019 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { 1020 Register len = op->len()->as_register(); 1021 1022 if (UseSlowPath || 1023 (!UseFastNewObjectArray && is_reference_type(op->type())) || 1024 (!UseFastNewTypeArray && !is_reference_type(op->type()))) { 1025 __ j(*op->stub()->entry()); 1026 } else { 1027 Register tmp1 = op->tmp1()->as_register(); 1028 Register tmp2 = op->tmp2()->as_register(); 1029 Register tmp3 = op->tmp3()->as_register(); 1030 if (len == tmp1) { 1031 tmp1 = tmp3; 1032 } else if (len == tmp2) { 1033 tmp2 = tmp3; 1034 } else if (len == tmp3) { 1035 // everything is ok 1036 } else { 1037 __ mv(tmp3, len); 1038 } 1039 __ allocate_array(op->obj()->as_register(), 1040 len, 1041 tmp1, 1042 tmp2, 1043 arrayOopDesc::header_size(op->type()), 1044 array_element_size(op->type()), 1045 op->klass()->as_register(), 1046 *op->stub()->entry()); 1047 } 1048 __ bind(*op->stub()->continuation()); 1049 } 1050 1051 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, ciProfileData *data, 1052 Register recv, Label* update_done) { 1053 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1054 Label next_test; 1055 // See if the receiver is receiver[n]. 1056 __ ld(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i)))); 1057 __ bne(recv, t1, next_test); 1058 Address data_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i))); 1059 __ increment(data_addr, DataLayout::counter_increment); 1060 __ j(*update_done); 1061 __ bind(next_test); 1062 } 1063 1064 // Didn't find receiver; find next empty slot and fill it in 1065 for (uint i = 0; i < ReceiverTypeData::row_limit(); i++) { 1066 Label next_test; 1067 Address recv_addr(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_offset(i))); 1068 __ ld(t1, recv_addr); 1069 __ bnez(t1, next_test); 1070 __ sd(recv, recv_addr); 1071 __ mv(t1, DataLayout::counter_increment); 1072 __ sd(t1, Address(mdo, md->byte_offset_of_slot(data, ReceiverTypeData::receiver_count_offset(i)))); 1073 __ j(*update_done); 1074 __ bind(next_test); 1075 } 1076 } 1077 1078 void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) { 1079 ciMethod* method = op->profiled_method(); 1080 assert(method != nullptr, "Should have method"); 1081 int bci = op->profiled_bci(); 1082 *md = method->method_data_or_null(); 1083 guarantee(*md != nullptr, "Sanity"); 1084 *data = ((*md)->bci_to_data(bci)); 1085 assert(*data != nullptr, "need data for type check"); 1086 assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1087 } 1088 1089 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1, 1090 Register k_RInfo, Register klass_RInfo, 1091 Label *failure_target, Label *success_target) { 1092 // get object class 1093 // not a safepoint as obj null check happens earlier 1094 __ load_klass(klass_RInfo, obj); 1095 if (k->is_loaded()) { 1096 // See if we get an immediate positive hit 1097 __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset()))); 1098 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) { 1099 __ bne(k_RInfo, t0, *failure_target, /* is_far */ true); 1100 // successful cast, fall through to profile or jump 1101 } else { 1102 // See if we get an immediate positive hit 1103 __ beq(k_RInfo, t0, *success_target); 1104 // check for self 1105 __ beq(klass_RInfo, k_RInfo, *success_target); 1106 1107 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo 1108 __ sd(k_RInfo, Address(sp, 0)); // sub klass 1109 __ sd(klass_RInfo, Address(sp, wordSize)); // super klass 1110 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1111 // load result to k_RInfo 1112 __ ld(k_RInfo, Address(sp, 0)); 1113 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo 1114 // result is a boolean 1115 __ beqz(k_RInfo, *failure_target, /* is_far */ true); 1116 // successful cast, fall through to profile or jump 1117 } 1118 } else { 1119 // perform the fast part of the checking logic 1120 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr); 1121 // call out-of-line instance of __ check_klass_subtytpe_slow_path(...) 1122 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo 1123 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass 1124 __ sd(k_RInfo, Address(sp, 0)); // super klass 1125 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 1126 // load result to k_RInfo 1127 __ ld(k_RInfo, Address(sp, 0)); 1128 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo 1129 // result is a boolean 1130 __ beqz(k_RInfo, *failure_target, /* is_far */ true); 1131 // successful cast, fall thriugh to profile or jump 1132 } 1133 } 1134 1135 void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Register obj, 1136 Register klass_RInfo, Label* obj_is_null) { 1137 Label not_null; 1138 __ bnez(obj, not_null); 1139 // Object is null, update MDO and exit 1140 Register mdo = klass_RInfo; 1141 __ mov_metadata(mdo, md->constant_encoding()); 1142 Address data_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset())); 1143 __ lbu(t0, data_addr); 1144 __ ori(t0, t0, BitData::null_seen_byte_constant()); 1145 __ sb(t0, data_addr); 1146 __ j(*obj_is_null); 1147 __ bind(not_null); 1148 } 1149 1150 void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) { 1151 if (!k->is_loaded()) { 1152 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1153 } else { 1154 __ mov_metadata(k_RInfo, k->constant_encoding()); 1155 } 1156 } 1157 1158 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { 1159 Register obj = op->object()->as_register(); 1160 Register k_RInfo = op->tmp1()->as_register(); 1161 Register klass_RInfo = op->tmp2()->as_register(); 1162 Register dst = op->result_opr()->as_register(); 1163 ciKlass* k = op->klass(); 1164 Register Rtmp1 = noreg; 1165 1166 // check if it needs to be profiled 1167 ciMethodData* md = nullptr; 1168 ciProfileData* data = nullptr; 1169 1170 const bool should_profile = op->should_profile(); 1171 if (should_profile) { 1172 data_check(op, &md, &data); 1173 } 1174 Label profile_cast_success, profile_cast_failure; 1175 Label *success_target = should_profile ? &profile_cast_success : success; 1176 Label *failure_target = should_profile ? &profile_cast_failure : failure; 1177 1178 if (obj == k_RInfo) { 1179 k_RInfo = dst; 1180 } else if (obj == klass_RInfo) { 1181 klass_RInfo = dst; 1182 } 1183 if (k->is_loaded() && !UseCompressedClassPointers) { 1184 select_different_registers(obj, dst, k_RInfo, klass_RInfo); 1185 } else { 1186 Rtmp1 = op->tmp3()->as_register(); 1187 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1); 1188 } 1189 1190 assert_different_registers(obj, k_RInfo, klass_RInfo); 1191 1192 if (should_profile) { 1193 profile_object(md, data, obj, klass_RInfo, obj_is_null); 1194 } else { 1195 __ beqz(obj, *obj_is_null); 1196 } 1197 1198 typecheck_loaded(op, k, k_RInfo); 1199 __ verify_oop(obj); 1200 1201 if (op->fast_check()) { 1202 // get object class 1203 // not a safepoint as obj null check happens earlier 1204 __ load_klass(t0, obj, t1); 1205 __ bne(t0, k_RInfo, *failure_target, /* is_far */ true); 1206 // successful cast, fall through to profile or jump 1207 } else { 1208 typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target); 1209 } 1210 if (should_profile) { 1211 type_profile(obj, md, klass_RInfo, k_RInfo, data, success, failure, profile_cast_success, profile_cast_failure); 1212 } 1213 __ j(*success); 1214 } 1215 1216 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { 1217 const bool should_profile = op->should_profile(); 1218 1219 LIR_Code code = op->code(); 1220 if (code == lir_store_check) { 1221 typecheck_lir_store(op, should_profile); 1222 } else if (code == lir_checkcast) { 1223 Register obj = op->object()->as_register(); 1224 Register dst = op->result_opr()->as_register(); 1225 Label success; 1226 emit_typecheck_helper(op, &success, op->stub()->entry(), &success); 1227 __ bind(success); 1228 if (dst != obj) { 1229 __ mv(dst, obj); 1230 } 1231 } else if (code == lir_instanceof) { 1232 Register obj = op->object()->as_register(); 1233 Register dst = op->result_opr()->as_register(); 1234 Label success, failure, done; 1235 emit_typecheck_helper(op, &success, &failure, &failure); 1236 __ bind(failure); 1237 __ mv(dst, zr); 1238 __ j(done); 1239 __ bind(success); 1240 __ mv(dst, 1); 1241 __ bind(done); 1242 } else { 1243 ShouldNotReachHere(); 1244 } 1245 } 1246 1247 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { 1248 assert(VM_Version::supports_cx8(), "wrong machine"); 1249 Register addr; 1250 if (op->addr()->is_register()) { 1251 addr = as_reg(op->addr()); 1252 } else { 1253 assert(op->addr()->is_address(), "what else?"); 1254 LIR_Address* addr_ptr = op->addr()->as_address_ptr(); 1255 assert(addr_ptr->disp() == 0, "need 0 disp"); 1256 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index"); 1257 addr = as_reg(addr_ptr->base()); 1258 } 1259 Register newval = as_reg(op->new_value()); 1260 Register cmpval = as_reg(op->cmp_value()); 1261 1262 if (op->code() == lir_cas_obj) { 1263 if (UseCompressedOops) { 1264 Register tmp1 = op->tmp1()->as_register(); 1265 assert(op->tmp1()->is_valid(), "must be"); 1266 Register tmp2 = op->tmp2()->as_register(); 1267 assert(op->tmp2()->is_valid(), "must be"); 1268 1269 __ encode_heap_oop(tmp1, cmpval); 1270 cmpval = tmp1; 1271 __ encode_heap_oop(tmp2, newval); 1272 newval = tmp2; 1273 caswu(addr, newval, cmpval); 1274 } else { 1275 casl(addr, newval, cmpval); 1276 } 1277 } else if (op->code() == lir_cas_int) { 1278 casw(addr, newval, cmpval); 1279 } else { 1280 casl(addr, newval, cmpval); 1281 } 1282 1283 if (op->result_opr()->is_valid()) { 1284 assert(op->result_opr()->is_register(), "need a register"); 1285 __ mv(as_reg(op->result_opr()), t0); // cas result in t0, and 0 for success 1286 } 1287 } 1288 1289 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) { 1290 switch (code) { 1291 case lir_abs: __ fabs_d(dest->as_double_reg(), value->as_double_reg()); break; 1292 case lir_sqrt: __ fsqrt_d(dest->as_double_reg(), value->as_double_reg()); break; 1293 default: ShouldNotReachHere(); 1294 } 1295 } 1296 1297 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) { 1298 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register"); 1299 Register Rleft = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 1300 if (dst->is_single_cpu()) { 1301 Register Rdst = dst->as_register(); 1302 if (right->is_constant()) { 1303 int right_const = right->as_jint(); 1304 if (Assembler::is_simm12(right_const)) { 1305 logic_op_imm(Rdst, Rleft, right_const, code); 1306 __ sign_extend(Rdst, Rdst, 32); 1307 } else { 1308 __ mv(t0, right_const); 1309 logic_op_reg32(Rdst, Rleft, t0, code); 1310 } 1311 } else { 1312 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo(); 1313 logic_op_reg32(Rdst, Rleft, Rright, code); 1314 } 1315 } else { 1316 Register Rdst = dst->as_register_lo(); 1317 if (right->is_constant()) { 1318 long right_const = right->as_jlong(); 1319 if (Assembler::is_simm12(right_const)) { 1320 logic_op_imm(Rdst, Rleft, right_const, code); 1321 } else { 1322 __ mv(t0, right_const); 1323 logic_op_reg(Rdst, Rleft, t0, code); 1324 } 1325 } else { 1326 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo(); 1327 logic_op_reg(Rdst, Rleft, Rright, code); 1328 } 1329 } 1330 } 1331 1332 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr src, LIR_Opr result, LIR_Op2* op) { 1333 ShouldNotCallThis(); 1334 } 1335 1336 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) { 1337 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { 1338 bool is_unordered_less = (code == lir_ucmp_fd2i); 1339 if (left->is_single_fpu()) { 1340 __ float_cmp(true, is_unordered_less ? -1 : 1, 1341 left->as_float_reg(), right->as_float_reg(), dst->as_register()); 1342 } else if (left->is_double_fpu()) { 1343 __ float_cmp(false, is_unordered_less ? -1 : 1, 1344 left->as_double_reg(), right->as_double_reg(), dst->as_register()); 1345 } else { 1346 ShouldNotReachHere(); 1347 } 1348 } else if (code == lir_cmp_l2i) { 1349 __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo()); 1350 } else { 1351 ShouldNotReachHere(); 1352 } 1353 } 1354 1355 void LIR_Assembler::align_call(LIR_Code code) { 1356 // With RVC a call instruction may get 2-byte aligned. 1357 // The address of the call instruction needs to be 4-byte aligned to 1358 // ensure that it does not span a cache line so that it can be patched. 1359 __ align(NativeInstruction::instruction_size); 1360 } 1361 1362 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { 1363 address call = __ trampoline_call(Address(op->addr(), rtype)); 1364 if (call == nullptr) { 1365 bailout("trampoline stub overflow"); 1366 return; 1367 } 1368 add_call_info(code_offset(), op->info()); 1369 __ post_call_nop(); 1370 } 1371 1372 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { 1373 address call = __ ic_call(op->addr()); 1374 if (call == nullptr) { 1375 bailout("trampoline stub overflow"); 1376 return; 1377 } 1378 add_call_info(code_offset(), op->info()); 1379 __ post_call_nop(); 1380 } 1381 1382 void LIR_Assembler::emit_static_call_stub() { 1383 address call_pc = __ pc(); 1384 MacroAssembler::assert_alignment(call_pc); 1385 address stub = __ start_a_stub(call_stub_size()); 1386 if (stub == nullptr) { 1387 bailout("static call stub overflow"); 1388 return; 1389 } 1390 1391 int start = __ offset(); 1392 1393 __ relocate(static_stub_Relocation::spec(call_pc)); 1394 __ emit_static_call_stub(); 1395 1396 assert(__ offset() - start + CompiledStaticCall::to_trampoline_stub_size() 1397 <= call_stub_size(), "stub too big"); 1398 __ end_a_stub(); 1399 } 1400 1401 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { 1402 assert(exceptionOop->as_register() == x10, "must match"); 1403 assert(exceptionPC->as_register() == x13, "must match"); 1404 1405 // exception object is not added to oop map by LinearScan 1406 // (LinearScan assumes that no oops are in fixed registers) 1407 info->add_register_oop(exceptionOop); 1408 Runtime1::StubID unwind_id; 1409 1410 // get current pc information 1411 // pc is only needed if the method has an exception handler, the unwind code does not need it. 1412 if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) { 1413 // As no instructions have been generated yet for this LIR node it's 1414 // possible that an oop map already exists for the current offset. 1415 // In that case insert an dummy NOP here to ensure all oop map PCs 1416 // are unique. See JDK-8237483. 1417 __ nop(); 1418 } 1419 int pc_for_athrow_offset = __ offset(); 1420 InternalAddress pc_for_athrow(__ pc()); 1421 __ relocate(pc_for_athrow.rspec(), [&] { 1422 int32_t offset; 1423 __ la_patchable(exceptionPC->as_register(), pc_for_athrow, offset); 1424 __ addi(exceptionPC->as_register(), exceptionPC->as_register(), offset); 1425 }); 1426 add_call_info(pc_for_athrow_offset, info); // for exception handler 1427 1428 __ verify_not_null_oop(x10); 1429 // search an exception handler (x10: exception oop, x13: throwing pc) 1430 if (compilation()->has_fpu_code()) { 1431 unwind_id = Runtime1::handle_exception_id; 1432 } else { 1433 unwind_id = Runtime1::handle_exception_nofpu_id; 1434 } 1435 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id))); 1436 __ nop(); 1437 } 1438 1439 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { 1440 assert(exceptionOop->as_register() == x10, "must match"); 1441 __ j(_unwind_handler_entry); 1442 } 1443 1444 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { 1445 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 1446 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 1447 Register count_reg = count->as_register(); 1448 if (dest->is_single_cpu()) { 1449 assert (dest->type() == T_INT, "unexpected result type"); 1450 assert (left->type() == T_INT, "unexpected left type"); 1451 __ andi(t0, count_reg, 31); // should not shift more than 31 bits 1452 switch (code) { 1453 case lir_shl: __ sllw(dest_reg, left_reg, t0); break; 1454 case lir_shr: __ sraw(dest_reg, left_reg, t0); break; 1455 case lir_ushr: __ srlw(dest_reg, left_reg, t0); break; 1456 default: ShouldNotReachHere(); 1457 } 1458 } else if (dest->is_double_cpu()) { 1459 __ andi(t0, count_reg, 63); // should not shift more than 63 bits 1460 switch (code) { 1461 case lir_shl: __ sll(dest_reg, left_reg, t0); break; 1462 case lir_shr: __ sra(dest_reg, left_reg, t0); break; 1463 case lir_ushr: __ srl(dest_reg, left_reg, t0); break; 1464 default: ShouldNotReachHere(); 1465 } 1466 } else { 1467 ShouldNotReachHere(); 1468 } 1469 } 1470 1471 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { 1472 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 1473 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 1474 if (dest->is_single_cpu()) { 1475 assert (dest->type() == T_INT, "unexpected result type"); 1476 assert (left->type() == T_INT, "unexpected left type"); 1477 count &= 0x1f; 1478 if (count != 0) { 1479 switch (code) { 1480 case lir_shl: __ slliw(dest_reg, left_reg, count); break; 1481 case lir_shr: __ sraiw(dest_reg, left_reg, count); break; 1482 case lir_ushr: __ srliw(dest_reg, left_reg, count); break; 1483 default: ShouldNotReachHere(); 1484 } 1485 } else { 1486 move_regs(left_reg, dest_reg); 1487 } 1488 } else if (dest->is_double_cpu()) { 1489 count &= 0x3f; 1490 if (count != 0) { 1491 switch (code) { 1492 case lir_shl: __ slli(dest_reg, left_reg, count); break; 1493 case lir_shr: __ srai(dest_reg, left_reg, count); break; 1494 case lir_ushr: __ srli(dest_reg, left_reg, count); break; 1495 default: ShouldNotReachHere(); 1496 } 1497 } else { 1498 move_regs(left->as_register_lo(), dest->as_register_lo()); 1499 } 1500 } else { 1501 ShouldNotReachHere(); 1502 } 1503 } 1504 1505 void LIR_Assembler::emit_lock(LIR_OpLock* op) { 1506 Register obj = op->obj_opr()->as_register(); // may not be an oop 1507 Register hdr = op->hdr_opr()->as_register(); 1508 Register lock = op->lock_opr()->as_register(); 1509 if (LockingMode == LM_MONITOR) { 1510 if (op->info() != nullptr) { 1511 add_debug_info_for_null_check_here(op->info()); 1512 __ null_check(obj); 1513 } 1514 __ j(*op->stub()->entry()); 1515 } else if (op->code() == lir_lock) { 1516 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 1517 // add debug info for NullPointerException only if one is possible 1518 int null_check_offset = __ lock_object(hdr, obj, lock, *op->stub()->entry()); 1519 if (op->info() != nullptr) { 1520 add_debug_info_for_null_check(null_check_offset, op->info()); 1521 } 1522 } else if (op->code() == lir_unlock) { 1523 assert(BasicLock::displaced_header_offset_in_bytes() == 0, "lock_reg must point to the displaced header"); 1524 __ unlock_object(hdr, obj, lock, *op->stub()->entry()); 1525 } else { 1526 Unimplemented(); 1527 } 1528 __ bind(*op->stub()->continuation()); 1529 } 1530 1531 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { 1532 Register obj = op->obj()->as_pointer_register(); 1533 Register result = op->result_opr()->as_pointer_register(); 1534 1535 CodeEmitInfo* info = op->info(); 1536 if (info != nullptr) { 1537 add_debug_info_for_null_check_here(info); 1538 } 1539 1540 if (UseCompressedClassPointers) { 1541 __ lwu(result, Address(obj, oopDesc::klass_offset_in_bytes())); 1542 __ decode_klass_not_null(result); 1543 } else { 1544 __ ld(result, Address(obj, oopDesc::klass_offset_in_bytes())); 1545 } 1546 } 1547 1548 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { 1549 ciMethod* method = op->profiled_method(); 1550 int bci = op->profiled_bci(); 1551 1552 // Update counter for all call types 1553 ciMethodData* md = method->method_data_or_null(); 1554 guarantee(md != nullptr, "Sanity"); 1555 ciProfileData* data = md->bci_to_data(bci); 1556 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls"); 1557 assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); 1558 Register mdo = op->mdo()->as_register(); 1559 __ mov_metadata(mdo, md->constant_encoding()); 1560 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1561 // Perform additional virtual call profiling for invokevirtual and 1562 // invokeinterface bytecodes 1563 if (op->should_profile_receiver_type()) { 1564 assert(op->recv()->is_single_cpu(), "recv must be allocated"); 1565 Register recv = op->recv()->as_register(); 1566 assert_different_registers(mdo, recv); 1567 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); 1568 ciKlass* known_klass = op->known_holder(); 1569 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) { 1570 // We know the type that will be seen at this call site; we can 1571 // statically update the MethodData* rather than needing to do 1572 // dynamic tests on the receiver type 1573 // NOTE: we should probably put a lock around this search to 1574 // avoid collisions by concurrent compilations 1575 ciVirtualCallData* vc_data = (ciVirtualCallData*) data; 1576 uint i; 1577 for (i = 0; i < VirtualCallData::row_limit(); i++) { 1578 ciKlass* receiver = vc_data->receiver(i); 1579 if (known_klass->equals(receiver)) { 1580 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 1581 __ increment(data_addr, DataLayout::counter_increment); 1582 return; 1583 } 1584 } 1585 1586 // Receiver type not found in profile data; select an empty slot 1587 // Note that this is less efficient than it should be because it 1588 // always does a write to the receiver part of the 1589 // VirtualCallData rather than just the first time 1590 for (i = 0; i < VirtualCallData::row_limit(); i++) { 1591 ciKlass* receiver = vc_data->receiver(i); 1592 if (receiver == nullptr) { 1593 Address recv_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_offset(i))); 1594 __ mov_metadata(t1, known_klass->constant_encoding()); 1595 __ sd(t1, recv_addr); 1596 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 1597 __ increment(data_addr, DataLayout::counter_increment); 1598 return; 1599 } 1600 } 1601 } else { 1602 __ load_klass(recv, recv); 1603 Label update_done; 1604 type_profile_helper(mdo, md, data, recv, &update_done); 1605 // Receiver did not match any saved receiver and there is no empty row for it. 1606 // Increment total counter to indicate polymorphic case. 1607 __ increment(counter_addr, DataLayout::counter_increment); 1608 1609 __ bind(update_done); 1610 } 1611 } else { 1612 // Static call 1613 __ increment(counter_addr, DataLayout::counter_increment); 1614 } 1615 } 1616 1617 void LIR_Assembler::emit_delay(LIR_OpDelay*) { Unimplemented(); } 1618 1619 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { 1620 __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); 1621 } 1622 1623 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { Unimplemented(); } 1624 1625 void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass, 1626 Register tmp, Label &next, Label &none, 1627 Address mdo_addr) { 1628 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) { 1629 if (exact_klass != nullptr) { 1630 __ mov_metadata(tmp, exact_klass->constant_encoding()); 1631 } else { 1632 __ load_klass(tmp, tmp); 1633 } 1634 1635 __ ld(t1, mdo_addr); 1636 __ xorr(tmp, tmp, t1); 1637 __ andi(t0, tmp, TypeEntries::type_klass_mask); 1638 // klass seen before, nothing to do. The unknown bit may have been 1639 // set already but no need to check. 1640 __ beqz(t0, next); 1641 1642 // already unknown. Nothing to do anymore. 1643 __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown)); 1644 __ bnez(t0, next); 1645 1646 if (TypeEntries::is_type_none(current_klass)) { 1647 __ beqz(t1, none); 1648 __ mv(t0, (u1)TypeEntries::null_seen); 1649 __ beq(t0, t1, none); 1650 // There is a chance that the checks above (re-reading profiling 1651 // data from memory) fail if another thread has just set the 1652 // profiling to this obj's klass 1653 __ membar(MacroAssembler::LoadLoad); 1654 __ ld(t1, mdo_addr); 1655 __ xorr(tmp, tmp, t1); 1656 __ andi(t0, tmp, TypeEntries::type_klass_mask); 1657 __ beqz(t0, next); 1658 } 1659 } else { 1660 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr && 1661 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only"); 1662 1663 __ ld(tmp, mdo_addr); 1664 // already unknown. Nothing to do anymore. 1665 __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown)); 1666 __ bnez(t0, next); 1667 } 1668 1669 // different than before. Cannot keep accurate profile. 1670 __ ld(t1, mdo_addr); 1671 __ ori(t1, t1, TypeEntries::type_unknown); 1672 __ sd(t1, mdo_addr); 1673 1674 if (TypeEntries::is_type_none(current_klass)) { 1675 __ j(next); 1676 1677 __ bind(none); 1678 // first time here. Set profile type. 1679 __ sd(tmp, mdo_addr); 1680 } 1681 } 1682 1683 void LIR_Assembler::check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp, 1684 Address mdo_addr, Label &next) { 1685 // There's a single possible klass at this profile point 1686 assert(exact_klass != nullptr, "should be"); 1687 if (TypeEntries::is_type_none(current_klass)) { 1688 __ mov_metadata(tmp, exact_klass->constant_encoding()); 1689 __ ld(t1, mdo_addr); 1690 __ xorr(tmp, tmp, t1); 1691 __ andi(t0, tmp, TypeEntries::type_klass_mask); 1692 __ beqz(t0, next); 1693 #ifdef ASSERT 1694 { 1695 Label ok; 1696 __ ld(t0, mdo_addr); 1697 __ beqz(t0, ok); 1698 __ mv(t1, (u1)TypeEntries::null_seen); 1699 __ beq(t0, t1, ok); 1700 // may have been set by another thread 1701 __ membar(MacroAssembler::LoadLoad); 1702 __ mov_metadata(t0, exact_klass->constant_encoding()); 1703 __ ld(t1, mdo_addr); 1704 __ xorr(t1, t0, t1); 1705 __ andi(t1, t1, TypeEntries::type_mask); 1706 __ beqz(t1, ok); 1707 1708 __ stop("unexpected profiling mismatch"); 1709 __ bind(ok); 1710 } 1711 #endif 1712 // first time here. Set profile type. 1713 __ sd(tmp, mdo_addr); 1714 } else { 1715 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr && 1716 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent"); 1717 1718 __ ld(tmp, mdo_addr); 1719 // already unknown. Nothing to do anymore. 1720 __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown)); 1721 __ bnez(t0, next); 1722 1723 __ ori(tmp, tmp, TypeEntries::type_unknown); 1724 __ sd(tmp, mdo_addr); 1725 } 1726 } 1727 1728 void LIR_Assembler::check_null(Register tmp, Label &update, intptr_t current_klass, 1729 Address mdo_addr, bool do_update, Label &next) { 1730 __ bnez(tmp, update); 1731 if (!TypeEntries::was_null_seen(current_klass)) { 1732 __ ld(t1, mdo_addr); 1733 __ ori(t1, t1, TypeEntries::null_seen); 1734 __ sd(t1, mdo_addr); 1735 } 1736 if (do_update) { 1737 __ j(next); 1738 } 1739 } 1740 1741 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { 1742 COMMENT("emit_profile_type {"); 1743 Register obj = op->obj()->as_register(); 1744 Register tmp = op->tmp()->as_pointer_register(); 1745 Address mdo_addr = as_Address(op->mdp()->as_address_ptr()); 1746 ciKlass* exact_klass = op->exact_klass(); 1747 intptr_t current_klass = op->current_klass(); 1748 bool not_null = op->not_null(); 1749 bool no_conflict = op->no_conflict(); 1750 1751 Label update, next, none; 1752 1753 bool do_null = !not_null; 1754 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass; 1755 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set; 1756 1757 assert(do_null || do_update, "why are we here?"); 1758 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?"); 1759 assert_different_registers(tmp, t0, t1, mdo_addr.base()); 1760 1761 __ verify_oop(obj); 1762 1763 if (tmp != obj) { 1764 __ mv(tmp, obj); 1765 } 1766 if (do_null) { 1767 check_null(tmp, update, current_klass, mdo_addr, do_update, next); 1768 #ifdef ASSERT 1769 } else { 1770 __ bnez(tmp, update); 1771 __ stop("unexpected null obj"); 1772 #endif 1773 } 1774 1775 __ bind(update); 1776 1777 if (do_update) { 1778 #ifdef ASSERT 1779 if (exact_klass != nullptr) { 1780 check_exact_klass(tmp, exact_klass); 1781 } 1782 #endif 1783 if (!no_conflict) { 1784 check_conflict(exact_klass, current_klass, tmp, next, none, mdo_addr); 1785 } else { 1786 check_no_conflict(exact_klass, current_klass, tmp, mdo_addr, next); 1787 } 1788 1789 __ bind(next); 1790 } 1791 COMMENT("} emit_profile_type"); 1792 } 1793 1794 void LIR_Assembler::align_backward_branch_target() { } 1795 1796 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) { 1797 // tmp must be unused 1798 assert(tmp->is_illegal(), "wasting a register if tmp is allocated"); 1799 1800 if (left->is_single_cpu()) { 1801 assert(dest->is_single_cpu(), "expect single result reg"); 1802 __ negw(dest->as_register(), left->as_register()); 1803 } else if (left->is_double_cpu()) { 1804 assert(dest->is_double_cpu(), "expect double result reg"); 1805 __ neg(dest->as_register_lo(), left->as_register_lo()); 1806 } else if (left->is_single_fpu()) { 1807 assert(dest->is_single_fpu(), "expect single float result reg"); 1808 __ fneg_s(dest->as_float_reg(), left->as_float_reg()); 1809 } else { 1810 assert(left->is_double_fpu(), "expect double float operand reg"); 1811 assert(dest->is_double_fpu(), "expect double float result reg"); 1812 __ fneg_d(dest->as_double_reg(), left->as_double_reg()); 1813 } 1814 } 1815 1816 1817 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 1818 if (patch_code != lir_patch_none) { 1819 deoptimize_trap(info); 1820 return; 1821 } 1822 1823 LIR_Address* adr = addr->as_address_ptr(); 1824 Register dst = dest->as_register_lo(); 1825 1826 assert_different_registers(dst, t0); 1827 if (adr->base()->is_valid() && dst == adr->base()->as_pointer_register() && (!adr->index()->is_cpu_register())) { 1828 int scale = adr->scale(); 1829 intptr_t offset = adr->disp(); 1830 LIR_Opr index_op = adr->index(); 1831 if (index_op->is_constant()) { 1832 offset += ((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale; 1833 } 1834 1835 if (!Assembler::is_simm12(offset)) { 1836 __ la(t0, as_Address(adr)); 1837 __ mv(dst, t0); 1838 return; 1839 } 1840 } 1841 1842 __ la(dst, as_Address(adr)); 1843 } 1844 1845 1846 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { 1847 assert(!tmp->is_valid(), "don't need temporary"); 1848 1849 CodeBlob *cb = CodeCache::find_blob(dest); 1850 if (cb != nullptr) { 1851 __ far_call(RuntimeAddress(dest)); 1852 } else { 1853 RuntimeAddress target(dest); 1854 __ relocate(target.rspec(), [&] { 1855 int32_t offset; 1856 __ la_patchable(t0, target, offset); 1857 __ jalr(x1, t0, offset); 1858 }); 1859 } 1860 1861 if (info != nullptr) { 1862 add_call_info_here(info); 1863 } 1864 __ post_call_nop(); 1865 } 1866 1867 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { 1868 if (dest->is_address() || src->is_address()) { 1869 move_op(src, dest, type, lir_patch_none, info, /* pop_fpu_stack */ false, /* wide */ false); 1870 } else { 1871 ShouldNotReachHere(); 1872 } 1873 } 1874 1875 #ifdef ASSERT 1876 // emit run-time assertion 1877 void LIR_Assembler::emit_assert(LIR_OpAssert* op) { 1878 assert(op->code() == lir_assert, "must be"); 1879 1880 Label ok; 1881 if (op->in_opr1()->is_valid()) { 1882 assert(op->in_opr2()->is_valid(), "both operands must be valid"); 1883 bool is_unordered = false; 1884 LIR_Condition cond = op->condition(); 1885 emit_branch(cond, op->in_opr1(), op->in_opr2(), ok, /* is_far */ false, 1886 /* is_unordered */(cond == lir_cond_greaterEqual || cond == lir_cond_greater) ? false : true); 1887 } else { 1888 assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); 1889 assert(op->condition() == lir_cond_always, "no other conditions allowed"); 1890 } 1891 1892 if (op->halt()) { 1893 const char* str = __ code_string(op->msg()); 1894 __ stop(str); 1895 } else { 1896 breakpoint(); 1897 } 1898 __ bind(ok); 1899 } 1900 #endif 1901 1902 #ifndef PRODUCT 1903 #define COMMENT(x) do { __ block_comment(x); } while (0) 1904 #else 1905 #define COMMENT(x) 1906 #endif 1907 1908 void LIR_Assembler::membar() { 1909 COMMENT("membar"); 1910 __ membar(MacroAssembler::AnyAny); 1911 } 1912 1913 void LIR_Assembler::membar_acquire() { 1914 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); 1915 } 1916 1917 void LIR_Assembler::membar_release() { 1918 __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); 1919 } 1920 1921 void LIR_Assembler::membar_loadload() { 1922 __ membar(MacroAssembler::LoadLoad); 1923 } 1924 1925 void LIR_Assembler::membar_storestore() { 1926 __ membar(MacroAssembler::StoreStore); 1927 } 1928 1929 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); } 1930 1931 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); } 1932 1933 void LIR_Assembler::on_spin_wait() { 1934 __ pause(); 1935 } 1936 1937 void LIR_Assembler::get_thread(LIR_Opr result_reg) { 1938 __ mv(result_reg->as_register(), xthread); 1939 } 1940 1941 void LIR_Assembler::peephole(LIR_List *lir) {} 1942 1943 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) { 1944 Address addr = as_Address(src->as_address_ptr()); 1945 BasicType type = src->type(); 1946 bool is_oop = is_reference_type(type); 1947 1948 get_op(type); 1949 1950 switch (code) { 1951 case lir_xadd: 1952 { 1953 RegisterOrConstant inc; 1954 Register tmp = as_reg(tmp_op); 1955 Register dst = as_reg(dest); 1956 if (data->is_constant()) { 1957 inc = RegisterOrConstant(as_long(data)); 1958 assert_different_registers(dst, addr.base(), tmp); 1959 assert_different_registers(tmp, t0); 1960 } else { 1961 inc = RegisterOrConstant(as_reg(data)); 1962 assert_different_registers(inc.as_register(), dst, addr.base(), tmp); 1963 } 1964 __ la(tmp, addr); 1965 (_masm->*add)(dst, inc, tmp); 1966 break; 1967 } 1968 case lir_xchg: 1969 { 1970 Register tmp = tmp_op->as_register(); 1971 Register obj = as_reg(data); 1972 Register dst = as_reg(dest); 1973 if (is_oop && UseCompressedOops) { 1974 __ encode_heap_oop(t0, obj); 1975 obj = t0; 1976 } 1977 assert_different_registers(obj, addr.base(), tmp); 1978 assert_different_registers(dst, addr.base(), tmp); 1979 __ la(tmp, addr); 1980 (_masm->*xchg)(dst, obj, tmp); 1981 if (is_oop && UseCompressedOops) { 1982 __ decode_heap_oop(dst); 1983 } 1984 } 1985 break; 1986 default: 1987 ShouldNotReachHere(); 1988 } 1989 __ membar(MacroAssembler::AnyAny); 1990 } 1991 1992 int LIR_Assembler::array_element_size(BasicType type) const { 1993 int elem_size = type2aelembytes(type); 1994 return exact_log2(elem_size); 1995 } 1996 1997 // helper functions which checks for overflow and sets bailout if it 1998 // occurs. Always returns a valid embeddable pointer but in the 1999 // bailout case the pointer won't be to unique storage. 2000 address LIR_Assembler::float_constant(float f) { 2001 address const_addr = __ float_constant(f); 2002 if (const_addr == nullptr) { 2003 bailout("const section overflow"); 2004 return __ code()->consts()->start(); 2005 } else { 2006 return const_addr; 2007 } 2008 } 2009 2010 address LIR_Assembler::double_constant(double d) { 2011 address const_addr = __ double_constant(d); 2012 if (const_addr == nullptr) { 2013 bailout("const section overflow"); 2014 return __ code()->consts()->start(); 2015 } else { 2016 return const_addr; 2017 } 2018 } 2019 2020 address LIR_Assembler::int_constant(jlong n) { 2021 address const_addr = __ long_constant(n); 2022 if (const_addr == nullptr) { 2023 bailout("const section overflow"); 2024 return __ code()->consts()->start(); 2025 } else { 2026 return const_addr; 2027 } 2028 } 2029 2030 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) { 2031 __ cmpxchg(addr, cmpval, newval, Assembler::int32, Assembler::aq /* acquire */, 2032 Assembler::rl /* release */, t0, true /* result as bool */); 2033 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1 2034 __ membar(MacroAssembler::AnyAny); 2035 } 2036 2037 void LIR_Assembler::caswu(Register addr, Register newval, Register cmpval) { 2038 __ cmpxchg(addr, cmpval, newval, Assembler::uint32, Assembler::aq /* acquire */, 2039 Assembler::rl /* release */, t0, true /* result as bool */); 2040 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1 2041 __ membar(MacroAssembler::AnyAny); 2042 } 2043 2044 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) { 2045 __ cmpxchg(addr, cmpval, newval, Assembler::int64, Assembler::aq /* acquire */, 2046 Assembler::rl /* release */, t0, true /* result as bool */); 2047 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1 2048 __ membar(MacroAssembler::AnyAny); 2049 } 2050 2051 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) { 2052 address target = nullptr; 2053 2054 switch (patching_id(info)) { 2055 case PatchingStub::access_field_id: 2056 target = Runtime1::entry_for(Runtime1::access_field_patching_id); 2057 break; 2058 case PatchingStub::load_klass_id: 2059 target = Runtime1::entry_for(Runtime1::load_klass_patching_id); 2060 break; 2061 case PatchingStub::load_mirror_id: 2062 target = Runtime1::entry_for(Runtime1::load_mirror_patching_id); 2063 break; 2064 case PatchingStub::load_appendix_id: 2065 target = Runtime1::entry_for(Runtime1::load_appendix_patching_id); 2066 break; 2067 default: ShouldNotReachHere(); 2068 } 2069 2070 __ far_call(RuntimeAddress(target)); 2071 add_call_info_here(info); 2072 } 2073 2074 void LIR_Assembler::check_exact_klass(Register tmp, ciKlass* exact_klass) { 2075 Label ok; 2076 __ load_klass(tmp, tmp); 2077 __ mov_metadata(t0, exact_klass->constant_encoding()); 2078 __ beq(tmp, t0, ok); 2079 __ stop("exact klass and actual klass differ"); 2080 __ bind(ok); 2081 } 2082 2083 void LIR_Assembler::get_op(BasicType type) { 2084 switch (type) { 2085 case T_INT: 2086 xchg = &MacroAssembler::atomic_xchgalw; 2087 add = &MacroAssembler::atomic_addalw; 2088 break; 2089 case T_LONG: 2090 xchg = &MacroAssembler::atomic_xchgal; 2091 add = &MacroAssembler::atomic_addal; 2092 break; 2093 case T_OBJECT: 2094 case T_ARRAY: 2095 if (UseCompressedOops) { 2096 xchg = &MacroAssembler::atomic_xchgalwu; 2097 add = &MacroAssembler::atomic_addalw; 2098 } else { 2099 xchg = &MacroAssembler::atomic_xchgal; 2100 add = &MacroAssembler::atomic_addal; 2101 } 2102 break; 2103 default: 2104 ShouldNotReachHere(); 2105 } 2106 } 2107 2108 // emit_opTypeCheck sub functions 2109 void LIR_Assembler::typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile) { 2110 Register value = op->object()->as_register(); 2111 Register array = op->array()->as_register(); 2112 Register k_RInfo = op->tmp1()->as_register(); 2113 Register klass_RInfo = op->tmp2()->as_register(); 2114 Register Rtmp1 = op->tmp3()->as_register(); 2115 2116 CodeStub* stub = op->stub(); 2117 2118 // check if it needs to be profiled 2119 ciMethodData* md = nullptr; 2120 ciProfileData* data = nullptr; 2121 2122 if (should_profile) { 2123 data_check(op, &md, &data); 2124 } 2125 Label profile_cast_success, profile_cast_failure, done; 2126 Label *success_target = should_profile ? &profile_cast_success : &done; 2127 Label *failure_target = should_profile ? &profile_cast_failure : stub->entry(); 2128 2129 if (should_profile) { 2130 profile_object(md, data, value, klass_RInfo, &done); 2131 } else { 2132 __ beqz(value, done); 2133 } 2134 2135 add_debug_info_for_null_check_here(op->info_for_exception()); 2136 __ load_klass(k_RInfo, array); 2137 __ load_klass(klass_RInfo, value); 2138 2139 lir_store_slowcheck(k_RInfo, klass_RInfo, Rtmp1, success_target, failure_target); 2140 2141 // fall through to the success case 2142 if (should_profile) { 2143 Register mdo = klass_RInfo; 2144 Register recv = k_RInfo; 2145 __ bind(profile_cast_success); 2146 __ mov_metadata(mdo, md->constant_encoding()); 2147 __ load_klass(recv, value); 2148 type_profile_helper(mdo, md, data, recv, &done); 2149 __ j(done); 2150 2151 __ bind(profile_cast_failure); 2152 __ mov_metadata(mdo, md->constant_encoding()); 2153 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 2154 __ ld(t1, counter_addr); 2155 __ addi(t1, t1, -DataLayout::counter_increment); 2156 __ sd(t1, counter_addr); 2157 __ j(*stub->entry()); 2158 } 2159 2160 __ bind(done); 2161 } 2162 2163 void LIR_Assembler::type_profile(Register obj, ciMethodData* md, Register klass_RInfo, Register k_RInfo, 2164 ciProfileData* data, Label* success, Label* failure, 2165 Label& profile_cast_success, Label& profile_cast_failure) { 2166 Register mdo = klass_RInfo; 2167 Register recv = k_RInfo; 2168 __ bind(profile_cast_success); 2169 __ mov_metadata(mdo, md->constant_encoding()); 2170 __ load_klass(recv, obj); 2171 Label update_done; 2172 type_profile_helper(mdo, md, data, recv, success); 2173 __ j(*success); 2174 2175 __ bind(profile_cast_failure); 2176 __ mov_metadata(mdo, md->constant_encoding()); 2177 Address counter_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 2178 __ ld(t0, counter_addr); 2179 __ addi(t0, t0, -DataLayout::counter_increment); 2180 __ sd(t0, counter_addr); 2181 __ j(*failure); 2182 } 2183 2184 void LIR_Assembler::lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1, 2185 Label* success_target, Label* failure_target) { 2186 // get instance klass (it's already uncompressed) 2187 __ ld(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); 2188 // perform the fast part of the checking logic 2189 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr); 2190 // call out-of-line instance of __ check_klass_subtype_slow_path(...) 2191 __ addi(sp, sp, -2 * wordSize); // 2: store k_RInfo and klass_RInfo 2192 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass 2193 __ sd(k_RInfo, Address(sp, 0)); // super klass 2194 __ far_call(RuntimeAddress(Runtime1::entry_for(Runtime1::slow_subtype_check_id))); 2195 // load result to k_RInfo 2196 __ ld(k_RInfo, Address(sp, 0)); 2197 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo 2198 // result is a boolean 2199 __ beqz(k_RInfo, *failure_target, /* is_far */ true); 2200 } 2201 2202 void LIR_Assembler::const2reg_helper(LIR_Opr src) { 2203 switch (src->as_constant_ptr()->type()) { 2204 case T_INT: 2205 case T_ADDRESS: 2206 case T_OBJECT: 2207 case T_ARRAY: 2208 case T_METADATA: 2209 const2reg(src, FrameMap::t0_opr, lir_patch_none, nullptr); 2210 break; 2211 case T_LONG: 2212 const2reg(src, FrameMap::t0_long_opr, lir_patch_none, nullptr); 2213 break; 2214 case T_FLOAT: 2215 case T_DOUBLE: 2216 default: 2217 ShouldNotReachHere(); 2218 } 2219 } 2220 2221 void LIR_Assembler::logic_op_reg32(Register dst, Register left, Register right, LIR_Code code) { 2222 switch (code) { 2223 case lir_logic_and: __ andrw(dst, left, right); break; 2224 case lir_logic_or: __ orrw (dst, left, right); break; 2225 case lir_logic_xor: __ xorrw(dst, left, right); break; 2226 default: ShouldNotReachHere(); 2227 } 2228 } 2229 2230 void LIR_Assembler::logic_op_reg(Register dst, Register left, Register right, LIR_Code code) { 2231 switch (code) { 2232 case lir_logic_and: __ andr(dst, left, right); break; 2233 case lir_logic_or: __ orr (dst, left, right); break; 2234 case lir_logic_xor: __ xorr(dst, left, right); break; 2235 default: ShouldNotReachHere(); 2236 } 2237 } 2238 2239 void LIR_Assembler::logic_op_imm(Register dst, Register left, int right, LIR_Code code) { 2240 switch (code) { 2241 case lir_logic_and: __ andi(dst, left, right); break; 2242 case lir_logic_or: __ ori (dst, left, right); break; 2243 case lir_logic_xor: __ xori(dst, left, right); break; 2244 default: ShouldNotReachHere(); 2245 } 2246 } 2247 2248 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) { 2249 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2250 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2251 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2252 __ sd(r, Address(sp, offset_from_rsp_in_bytes)); 2253 } 2254 2255 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) { 2256 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2257 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2258 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2259 __ mv(t0, c); 2260 __ sd(t0, Address(sp, offset_from_rsp_in_bytes)); 2261 } 2262 2263 #undef __