1 /* 2 * Copyright (c) 2000, 2026, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. 4 * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "asm/assembler.hpp" 28 #include "asm/macroAssembler.inline.hpp" 29 #include "c1/c1_CodeStubs.hpp" 30 #include "c1/c1_Compilation.hpp" 31 #include "c1/c1_LIRAssembler.hpp" 32 #include "c1/c1_MacroAssembler.hpp" 33 #include "c1/c1_Runtime1.hpp" 34 #include "c1/c1_ValueStack.hpp" 35 #include "ci/ciArrayKlass.hpp" 36 #include "ci/ciInlineKlass.hpp" 37 #include "ci/ciInstance.hpp" 38 #include "ci/ciObjArrayKlass.hpp" 39 #include "code/compiledIC.hpp" 40 #include "gc/shared/collectedHeap.hpp" 41 #include "nativeInst_riscv.hpp" 42 #include "oops/objArrayKlass.hpp" 43 #include "oops/oop.inline.hpp" 44 #include "runtime/frame.inline.hpp" 45 #include "runtime/sharedRuntime.hpp" 46 #include "utilities/powerOfTwo.hpp" 47 #include "vmreg_riscv.inline.hpp" 48 49 #ifndef PRODUCT 50 #define COMMENT(x) do { __ block_comment(x); } while (0) 51 #else 52 #define COMMENT(x) 53 #endif 54 55 NEEDS_CLEANUP // remove this definitions ? 56 const Register SYNC_header = x10; // synchronization header 57 const Register SHIFT_count = x10; // where count for shift operations must be 58 59 #define __ _masm-> 60 61 static void select_different_registers(Register preserve, 62 Register extra, 63 Register &tmp1, 64 Register &tmp2, 65 Register &tmp3) { 66 if (tmp1 == preserve) { 67 assert_different_registers(tmp1, tmp2, tmp3, extra); 68 tmp1 = extra; 69 } else if (tmp2 == preserve) { 70 assert_different_registers(tmp1, tmp2, tmp3, extra); 71 tmp2 = extra; 72 } else if (tmp3 == preserve) { 73 assert_different_registers(tmp1, tmp2, tmp3, extra); 74 tmp3 = extra; 75 } 76 assert_different_registers(preserve, tmp1, tmp2, tmp3); 77 } 78 79 bool LIR_Assembler::is_small_constant(LIR_Opr opr) { Unimplemented(); return false; } 80 81 void LIR_Assembler::clinit_barrier(ciMethod* method) { 82 assert(VM_Version::supports_fast_class_init_checks(), "sanity"); 83 assert(!method->holder()->is_not_initialized(), "initialization should have been started"); 84 85 Label L_skip_barrier; 86 87 __ mov_metadata(t1, method->holder()->constant_encoding()); 88 __ clinit_barrier(t1, t0, &L_skip_barrier /* L_fast_path */); 89 __ far_jump(RuntimeAddress(SharedRuntime::get_handle_wrong_method_stub())); 90 __ bind(L_skip_barrier); 91 } 92 93 LIR_Opr LIR_Assembler::receiverOpr() { 94 return FrameMap::receiver_opr; 95 } 96 97 LIR_Opr LIR_Assembler::osrBufferPointer() { 98 return FrameMap::as_pointer_opr(receiverOpr()->as_register()); 99 } 100 101 void LIR_Assembler::breakpoint() { Unimplemented(); } 102 103 void LIR_Assembler::push(LIR_Opr opr) { Unimplemented(); } 104 105 void LIR_Assembler::pop(LIR_Opr opr) { Unimplemented(); } 106 107 static jlong as_long(LIR_Opr data) { 108 jlong result; 109 switch (data->type()) { 110 case T_INT: 111 result = (data->as_jint()); 112 break; 113 case T_LONG: 114 result = (data->as_jlong()); 115 break; 116 default: 117 ShouldNotReachHere(); 118 result = 0; // unreachable 119 } 120 return result; 121 } 122 123 Address LIR_Assembler::as_Address(LIR_Address* addr, Register tmp) { 124 if (addr->base()->is_illegal()) { 125 assert(addr->index()->is_illegal(), "must be illegal too"); 126 __ movptr(tmp, (address)addr->disp()); 127 return Address(tmp, 0); 128 } 129 130 Register base = addr->base()->as_pointer_register(); 131 LIR_Opr index_opr = addr->index(); 132 133 if (index_opr->is_illegal()) { 134 return Address(base, addr->disp()); 135 } 136 137 int scale = addr->scale(); 138 if (index_opr->is_cpu_register()) { 139 Register index; 140 if (index_opr->is_single_cpu()) { 141 index = index_opr->as_register(); 142 } else { 143 index = index_opr->as_register_lo(); 144 } 145 if (scale != 0) { 146 __ shadd(tmp, index, base, tmp, scale); 147 } else { 148 __ add(tmp, base, index); 149 } 150 return Address(tmp, addr->disp()); 151 } else if (index_opr->is_constant()) { 152 intptr_t addr_offset = (((intptr_t)index_opr->as_constant_ptr()->as_jint()) << scale) + addr->disp(); 153 return Address(base, addr_offset); 154 } 155 156 Unimplemented(); 157 return Address(); 158 } 159 160 Address LIR_Assembler::as_Address_hi(LIR_Address* addr) { 161 ShouldNotReachHere(); 162 return Address(); 163 } 164 165 Address LIR_Assembler::as_Address(LIR_Address* addr) { 166 return as_Address(addr, t0); 167 } 168 169 Address LIR_Assembler::as_Address_lo(LIR_Address* addr) { 170 return as_Address(addr); 171 } 172 173 // Ensure a valid Address (base + offset) to a stack-slot. If stack access is 174 // not encodable as a base + (immediate) offset, generate an explicit address 175 // calculation to hold the address in t0. 176 Address LIR_Assembler::stack_slot_address(int index, uint size, int adjust) { 177 precond(size == 4 || size == 8); 178 Address addr = frame_map()->address_for_slot(index, adjust); 179 precond(addr.getMode() == Address::base_plus_offset); 180 precond(addr.base() == sp); 181 precond(addr.offset() > 0); 182 uint mask = size - 1; 183 assert((addr.offset() & mask) == 0, "scaled offsets only"); 184 185 return addr; 186 } 187 188 void LIR_Assembler::osr_entry() { 189 offsets()->set_value(CodeOffsets::OSR_Entry, code_offset()); 190 BlockBegin* osr_entry = compilation()->hir()->osr_entry(); 191 guarantee(osr_entry != nullptr, "null osr_entry!"); 192 ValueStack* entry_state = osr_entry->state(); 193 int number_of_locks = entry_state->locks_size(); 194 195 // we jump here if osr happens with the interpreter 196 // state set up to continue at the beginning of the 197 // loop that triggered osr - in particular, we have 198 // the following registers setup: 199 // 200 // x12: osr buffer 201 // 202 203 //build frame 204 ciMethod* m = compilation()->method(); 205 __ build_frame(initial_frame_size_in_bytes(), bang_size_in_bytes()); 206 207 // OSR buffer is 208 // 209 // locals[nlocals-1..0] 210 // monitors[0..number_of_locks] 211 // 212 // locals is a direct copy of the interpreter frame so in the osr buffer 213 // so first slot in the local array is the last local from the interpreter 214 // and last slot is local[0] (receiver) from the interpreter 215 // 216 // Similarly with locks. The first lock slot in the osr buffer is the nth lock 217 // from the interpreter frame, the nth lock slot in the osr buffer is 0th lock 218 // in the interpreter frame (the method lock if a sync method) 219 220 // Initialize monitors in the compiled activation. 221 // x12: pointer to osr buffer 222 // All other registers are dead at this point and the locals will be 223 // copied into place by code emitted in the IR. 224 225 Register OSR_buf = osrBufferPointer()->as_pointer_register(); 226 { 227 assert(frame::interpreter_frame_monitor_size() == BasicObjectLock::size(), "adjust code below"); 228 int monitor_offset = BytesPerWord * method()->max_locals() + 229 (2 * BytesPerWord) * (number_of_locks - 1); 230 // SharedRuntime::OSR_migration_begin() packs BasicObjectLocks in 231 // the OSR buffer using 2 word entries: first the lock and then 232 // the oop. 233 for (int i = 0; i < number_of_locks; i++) { 234 int slot_offset = monitor_offset - ((i * 2) * BytesPerWord); 235 #ifdef ASSERT 236 // verify the interpreter's monitor has a non-null object 237 { 238 Label L; 239 __ ld(t0, Address(OSR_buf, slot_offset + 1 * BytesPerWord)); 240 __ bnez(t0, L); 241 __ stop("locked object is null"); 242 __ bind(L); 243 } 244 #endif // ASSERT 245 __ ld(x9, Address(OSR_buf, slot_offset + 0)); 246 __ sd(x9, frame_map()->address_for_monitor_lock(i)); 247 __ ld(x9, Address(OSR_buf, slot_offset + 1 * BytesPerWord)); 248 __ sd(x9, frame_map()->address_for_monitor_object(i)); 249 } 250 } 251 } 252 253 // inline cache check; done before the frame is built. 254 int LIR_Assembler::check_icache() { 255 return __ ic_check(CodeEntryAlignment); 256 } 257 258 void LIR_Assembler::jobject2reg(jobject o, Register reg) { 259 if (o == nullptr) { 260 __ mv(reg, zr); 261 } else { 262 __ movoop(reg, o); 263 } 264 } 265 266 void LIR_Assembler::jobject2reg_with_patching(Register reg, CodeEmitInfo *info) { 267 deoptimize_trap(info); 268 } 269 270 // This specifies the rsp decrement needed to build the frame 271 int LIR_Assembler::initial_frame_size_in_bytes() const { 272 // if rounding, must let FrameMap know! 273 274 return in_bytes(frame_map()->framesize_in_bytes()); 275 } 276 277 int LIR_Assembler::emit_exception_handler() { 278 // generate code for exception handler 279 address handler_base = __ start_a_stub(exception_handler_size()); 280 if (handler_base == nullptr) { 281 // not enough space left for the handler 282 bailout("exception handler overflow"); 283 return -1; 284 } 285 286 int offset = code_offset(); 287 288 // the exception oop and pc are in x10, and x13 289 // no other registers need to be preserved, so invalidate them 290 __ invalidate_registers(false, true, true, false, true, true); 291 292 // check that there is really an exception 293 __ verify_not_null_oop(x10); 294 295 // search an exception handler (x10: exception oop, x13: throwing pc) 296 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_handle_exception_from_callee_id))); 297 __ should_not_reach_here(); 298 guarantee(code_offset() - offset <= exception_handler_size(), "overflow"); 299 __ end_a_stub(); 300 301 return offset; 302 } 303 304 // Emit the code to remove the frame from the stack in the exception 305 // unwind path. 306 int LIR_Assembler::emit_unwind_handler() { 307 #ifndef PRODUCT 308 if (CommentedAssembly) { 309 _masm->block_comment("Unwind handler"); 310 } 311 #endif // PRODUCT 312 313 int offset = code_offset(); 314 315 // Fetch the exception from TLS and clear out exception related thread state 316 __ ld(x10, Address(xthread, JavaThread::exception_oop_offset())); 317 __ sd(zr, Address(xthread, JavaThread::exception_oop_offset())); 318 __ sd(zr, Address(xthread, JavaThread::exception_pc_offset())); 319 320 __ bind(_unwind_handler_entry); 321 __ verify_not_null_oop(x10); 322 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 323 __ mv(x9, x10); // Preserve the exception 324 } 325 326 // Perform needed unlocking 327 MonitorExitStub* stub = nullptr; 328 if (method()->is_synchronized()) { 329 monitor_address(0, FrameMap::r10_opr); 330 stub = new MonitorExitStub(FrameMap::r10_opr, 0); 331 __ unlock_object(x15, x14, x10, x16, *stub->entry()); 332 __ bind(*stub->continuation()); 333 } 334 335 if (compilation()->env()->dtrace_method_probes()) { 336 __ mv(c_rarg0, xthread); 337 __ mov_metadata(c_rarg1, method()->constant_encoding()); 338 __ call_VM_leaf(CAST_FROM_FN_PTR(address, SharedRuntime::dtrace_method_exit), c_rarg0, c_rarg1); 339 } 340 341 if (method()->is_synchronized() || compilation()->env()->dtrace_method_probes()) { 342 __ mv(x10, x9); // Restore the exception 343 } 344 345 // remove the activation and dispatch to the unwind handler 346 __ block_comment("remove_frame and dispatch to the unwind handler"); 347 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair()); 348 __ far_jump(RuntimeAddress(Runtime1::entry_for(StubId::c1_unwind_exception_id))); 349 350 // Emit the slow path assembly 351 if (stub != nullptr) { 352 stub->emit_code(this); 353 } 354 355 return offset; 356 } 357 358 int LIR_Assembler::emit_deopt_handler() { 359 // generate code for exception handler 360 address handler_base = __ start_a_stub(deopt_handler_size()); 361 if (handler_base == nullptr) { 362 // not enough space left for the handler 363 bailout("deopt handler overflow"); 364 return -1; 365 } 366 367 int offset = code_offset(); 368 369 Label start; 370 __ bind(start); 371 372 __ far_call(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); 373 374 int entry_offset = __ offset(); 375 __ j(start); 376 377 guarantee(code_offset() - offset <= deopt_handler_size(), "overflow"); 378 assert(code_offset() - entry_offset >= NativePostCallNop::first_check_size, 379 "out of bounds read in post-call NOP check"); 380 __ end_a_stub(); 381 382 return entry_offset; 383 } 384 385 void LIR_Assembler::return_op(LIR_Opr result, C1SafepointPollStub* code_stub) { 386 assert(result->is_illegal() || !result->is_single_cpu() || result->as_register() == x10, "word returns are in x10"); 387 388 assert(!InlineTypeReturnedAsFields, "unimplemented"); 389 390 // Pop the stack before the safepoint code 391 __ remove_frame(initial_frame_size_in_bytes(), needs_stack_repair()); 392 393 if (StackReservedPages > 0 && compilation()->has_reserved_stack_access()) { 394 __ reserved_stack_check(); 395 } 396 397 code_stub->set_safepoint_offset(__ offset()); 398 __ relocate(relocInfo::poll_return_type); 399 __ safepoint_poll(*code_stub->entry(), true /* at_return */, true /* in_nmethod */); 400 __ ret(); 401 } 402 403 int LIR_Assembler::store_inline_type_fields_to_buf(ciInlineKlass* vk) { 404 Unimplemented(); 405 return 0; 406 } 407 408 int LIR_Assembler::safepoint_poll(LIR_Opr tmp, CodeEmitInfo* info) { 409 guarantee(info != nullptr, "Shouldn't be null"); 410 __ get_polling_page(t0, relocInfo::poll_type); 411 add_debug_info_for_branch(info); // This isn't just debug info: 412 // it's the oop map 413 __ read_polling_page(t0, 0, relocInfo::poll_type); 414 return __ offset(); 415 } 416 417 void LIR_Assembler::move_regs(Register from_reg, Register to_reg) { 418 __ mv(to_reg, from_reg); 419 } 420 421 void LIR_Assembler::swap_reg(Register a, Register b) { Unimplemented(); } 422 423 void LIR_Assembler::const2reg(LIR_Opr src, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 424 assert(src->is_constant(), "should not call otherwise"); 425 assert(dest->is_register(), "should not call otherwise"); 426 LIR_Const* c = src->as_constant_ptr(); 427 address const_addr = nullptr; 428 jfloat fconst; 429 jdouble dconst; 430 431 switch (c->type()) { 432 case T_INT: 433 assert(patch_code == lir_patch_none, "no patching handled here"); 434 __ mv(dest->as_register(), c->as_jint()); 435 break; 436 437 case T_ADDRESS: 438 assert(patch_code == lir_patch_none, "no patching handled here"); 439 __ mv(dest->as_register(), c->as_jint()); 440 break; 441 442 case T_LONG: 443 assert(patch_code == lir_patch_none, "no patching handled here"); 444 __ mv(dest->as_register_lo(), (intptr_t)c->as_jlong()); 445 break; 446 447 case T_OBJECT: 448 case T_ARRAY: 449 if (patch_code != lir_patch_none) { 450 jobject2reg_with_patching(dest->as_register(), info); 451 } else { 452 jobject2reg(c->as_jobject(), dest->as_register()); 453 } 454 break; 455 456 case T_METADATA: 457 if (patch_code != lir_patch_none) { 458 klass2reg_with_patching(dest->as_register(), info); 459 } else { 460 __ mov_metadata(dest->as_register(), c->as_metadata()); 461 } 462 break; 463 464 case T_FLOAT: 465 fconst = c->as_jfloat(); 466 if (MacroAssembler::can_fp_imm_load(fconst)) { 467 __ fli_s(dest->as_float_reg(), fconst); 468 } else { 469 const_addr = float_constant(fconst); 470 assert(const_addr != nullptr, "must create float constant in the constant table"); 471 __ flw(dest->as_float_reg(), InternalAddress(const_addr)); 472 } 473 break; 474 475 case T_DOUBLE: 476 dconst = c->as_jdouble(); 477 if (MacroAssembler::can_dp_imm_load(dconst)) { 478 __ fli_d(dest->as_double_reg(), dconst); 479 } else { 480 const_addr = double_constant(c->as_jdouble()); 481 assert(const_addr != nullptr, "must create double constant in the constant table"); 482 __ fld(dest->as_double_reg(), InternalAddress(const_addr)); 483 } 484 break; 485 486 default: 487 ShouldNotReachHere(); 488 } 489 } 490 491 void LIR_Assembler::const2stack(LIR_Opr src, LIR_Opr dest) { 492 assert(src->is_constant(), "should not call otherwise"); 493 assert(dest->is_stack(), "should not call otherwise"); 494 LIR_Const* c = src->as_constant_ptr(); 495 switch (c->type()) { 496 case T_OBJECT: 497 if (c->as_jobject() == nullptr) { 498 __ sd(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 499 } else { 500 const2reg(src, FrameMap::t1_opr, lir_patch_none, nullptr); 501 reg2stack(FrameMap::t1_opr, dest, c->type()); 502 } 503 break; 504 case T_ADDRESS: // fall through 505 const2reg(src, FrameMap::t1_opr, lir_patch_none, nullptr); 506 reg2stack(FrameMap::t1_opr, dest, c->type()); 507 case T_INT: // fall through 508 case T_FLOAT: 509 if (c->as_jint_bits() == 0) { 510 __ sw(zr, frame_map()->address_for_slot(dest->single_stack_ix())); 511 } else { 512 __ mv(t1, c->as_jint_bits()); 513 __ sw(t1, frame_map()->address_for_slot(dest->single_stack_ix())); 514 } 515 break; 516 case T_LONG: // fall through 517 case T_DOUBLE: 518 if (c->as_jlong_bits() == 0) { 519 __ sd(zr, frame_map()->address_for_slot(dest->double_stack_ix(), 520 lo_word_offset_in_bytes)); 521 } else { 522 __ mv(t1, (intptr_t)c->as_jlong_bits()); 523 __ sd(t1, frame_map()->address_for_slot(dest->double_stack_ix(), 524 lo_word_offset_in_bytes)); 525 } 526 break; 527 default: 528 ShouldNotReachHere(); 529 } 530 } 531 532 void LIR_Assembler::const2mem(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info, bool wide) { 533 assert(src->is_constant(), "should not call otherwise"); 534 assert(dest->is_address(), "should not call otherwise"); 535 LIR_Const* c = src->as_constant_ptr(); 536 LIR_Address* to_addr = dest->as_address_ptr(); 537 void (MacroAssembler::* insn)(Register Rt, const Address &adr, Register temp); 538 switch (type) { 539 case T_ADDRESS: 540 assert(c->as_jint() == 0, "should be"); 541 insn = &MacroAssembler::sd; break; 542 case T_LONG: 543 assert(c->as_jlong() == 0, "should be"); 544 insn = &MacroAssembler::sd; break; 545 case T_DOUBLE: 546 assert(c->as_jdouble() == 0.0, "should be"); 547 insn = &MacroAssembler::sd; break; 548 case T_INT: 549 assert(c->as_jint() == 0, "should be"); 550 insn = &MacroAssembler::sw; break; 551 case T_FLOAT: 552 assert(c->as_jfloat() == 0.0f, "should be"); 553 insn = &MacroAssembler::sw; break; 554 case T_OBJECT: // fall through 555 case T_ARRAY: 556 assert(c->as_jobject() == nullptr, "should be"); 557 if (UseCompressedOops && !wide) { 558 insn = &MacroAssembler::sw; 559 } else { 560 insn = &MacroAssembler::sd; 561 } 562 break; 563 case T_CHAR: // fall through 564 case T_SHORT: 565 assert(c->as_jint() == 0, "should be"); 566 insn = &MacroAssembler::sh; 567 break; 568 case T_BOOLEAN: // fall through 569 case T_BYTE: 570 assert(c->as_jint() == 0, "should be"); 571 insn = &MacroAssembler::sb; break; 572 default: 573 ShouldNotReachHere(); 574 insn = &MacroAssembler::sd; // unreachable 575 } 576 if (info != nullptr) { 577 add_debug_info_for_null_check_here(info); 578 } 579 (_masm->*insn)(zr, as_Address(to_addr), t0); 580 } 581 582 void LIR_Assembler::reg2reg(LIR_Opr src, LIR_Opr dest) { 583 assert(src->is_register(), "should not call otherwise"); 584 assert(dest->is_register(), "should not call otherwise"); 585 586 // move between cpu-registers 587 if (dest->is_single_cpu()) { 588 if (src->type() == T_LONG) { 589 // Can do LONG -> OBJECT 590 move_regs(src->as_register_lo(), dest->as_register()); 591 return; 592 } 593 assert(src->is_single_cpu(), "must match"); 594 if (src->type() == T_OBJECT) { 595 __ verify_oop(src->as_register()); 596 } 597 move_regs(src->as_register(), dest->as_register()); 598 } else if (dest->is_double_cpu()) { 599 if (is_reference_type(src->type())) { 600 __ verify_oop(src->as_register()); 601 move_regs(src->as_register(), dest->as_register_lo()); 602 return; 603 } 604 assert(src->is_double_cpu(), "must match"); 605 Register f_lo = src->as_register_lo(); 606 Register f_hi = src->as_register_hi(); 607 Register t_lo = dest->as_register_lo(); 608 Register t_hi = dest->as_register_hi(); 609 assert(f_hi == f_lo, "must be same"); 610 assert(t_hi == t_lo, "must be same"); 611 move_regs(f_lo, t_lo); 612 } else if (dest->is_single_fpu()) { 613 assert(src->is_single_fpu(), "expect single fpu"); 614 __ fmv_s(dest->as_float_reg(), src->as_float_reg()); 615 } else if (dest->is_double_fpu()) { 616 assert(src->is_double_fpu(), "expect double fpu"); 617 __ fmv_d(dest->as_double_reg(), src->as_double_reg()); 618 } else { 619 ShouldNotReachHere(); 620 } 621 } 622 623 void LIR_Assembler::reg2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 624 precond(src->is_register() && dest->is_stack()); 625 626 uint const c_sz32 = sizeof(uint32_t); 627 uint const c_sz64 = sizeof(uint64_t); 628 629 assert(src->is_register(), "should not call otherwise"); 630 assert(dest->is_stack(), "should not call otherwise"); 631 if (src->is_single_cpu()) { 632 int index = dest->single_stack_ix(); 633 if (is_reference_type(type)) { 634 __ sd(src->as_register(), stack_slot_address(index, c_sz64)); 635 __ verify_oop(src->as_register()); 636 } else if (type == T_METADATA || type == T_DOUBLE || type == T_ADDRESS) { 637 __ sd(src->as_register(), stack_slot_address(index, c_sz64)); 638 } else { 639 __ sw(src->as_register(), stack_slot_address(index, c_sz32)); 640 } 641 } else if (src->is_double_cpu()) { 642 int index = dest->double_stack_ix(); 643 Address dest_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes); 644 __ sd(src->as_register_lo(), dest_addr_LO); 645 } else if (src->is_single_fpu()) { 646 int index = dest->single_stack_ix(); 647 __ fsw(src->as_float_reg(), stack_slot_address(index, c_sz32)); 648 } else if (src->is_double_fpu()) { 649 int index = dest->double_stack_ix(); 650 __ fsd(src->as_double_reg(), stack_slot_address(index, c_sz64)); 651 } else { 652 ShouldNotReachHere(); 653 } 654 } 655 656 void LIR_Assembler::reg2mem(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) { 657 LIR_Address* to_addr = dest->as_address_ptr(); 658 // t0 was used as tmp reg in as_Address, so we use t1 as compressed_src 659 Register compressed_src = t1; 660 661 if (patch_code != lir_patch_none) { 662 deoptimize_trap(info); 663 return; 664 } 665 666 if (is_reference_type(type)) { 667 __ verify_oop(src->as_register()); 668 669 if (UseCompressedOops && !wide) { 670 __ encode_heap_oop(compressed_src, src->as_register()); 671 } else { 672 compressed_src = src->as_register(); 673 } 674 } 675 676 int null_check_here = code_offset(); 677 678 switch (type) { 679 case T_FLOAT: 680 __ fsw(src->as_float_reg(), as_Address(to_addr)); 681 break; 682 683 case T_DOUBLE: 684 __ fsd(src->as_double_reg(), as_Address(to_addr)); 685 break; 686 687 case T_ARRAY: // fall through 688 case T_OBJECT: 689 if (UseCompressedOops && !wide) { 690 __ sw(compressed_src, as_Address(to_addr)); 691 } else { 692 __ sd(compressed_src, as_Address(to_addr)); 693 } 694 break; 695 case T_METADATA: 696 // We get here to store a method pointer to the stack to pass to 697 // a dtrace runtime call. This can't work on 64 bit with 698 // compressed klass ptrs: T_METADATA can be compressed klass 699 // ptr or a 64 bit method pointer. 700 ShouldNotReachHere(); 701 __ sd(src->as_register(), as_Address(to_addr)); 702 break; 703 case T_ADDRESS: 704 __ sd(src->as_register(), as_Address(to_addr)); 705 break; 706 case T_INT: 707 __ sw(src->as_register(), as_Address(to_addr)); 708 break; 709 case T_LONG: 710 __ sd(src->as_register_lo(), as_Address(to_addr)); 711 break; 712 case T_BYTE: // fall through 713 case T_BOOLEAN: 714 __ sb(src->as_register(), as_Address(to_addr)); 715 break; 716 case T_CHAR: // fall through 717 case T_SHORT: 718 __ sh(src->as_register(), as_Address(to_addr)); 719 break; 720 default: 721 ShouldNotReachHere(); 722 } 723 724 if (info != nullptr) { 725 add_debug_info_for_null_check(null_check_here, info); 726 } 727 } 728 729 void LIR_Assembler::stack2reg(LIR_Opr src, LIR_Opr dest, BasicType type) { 730 precond(src->is_stack() && dest->is_register()); 731 732 uint const c_sz32 = sizeof(uint32_t); 733 uint const c_sz64 = sizeof(uint64_t); 734 735 if (dest->is_single_cpu()) { 736 int index = src->single_stack_ix(); 737 if (type == T_INT) { 738 __ lw(dest->as_register(), stack_slot_address(index, c_sz32)); 739 } else if (is_reference_type(type)) { 740 __ ld(dest->as_register(), stack_slot_address(index, c_sz64)); 741 __ verify_oop(dest->as_register()); 742 } else if (type == T_METADATA || type == T_ADDRESS) { 743 __ ld(dest->as_register(), stack_slot_address(index, c_sz64)); 744 } else { 745 __ lwu(dest->as_register(), stack_slot_address(index, c_sz32)); 746 } 747 } else if (dest->is_double_cpu()) { 748 int index = src->double_stack_ix(); 749 Address src_addr_LO = stack_slot_address(index, c_sz64, lo_word_offset_in_bytes); 750 __ ld(dest->as_register_lo(), src_addr_LO); 751 } else if (dest->is_single_fpu()) { 752 int index = src->single_stack_ix(); 753 __ flw(dest->as_float_reg(), stack_slot_address(index, c_sz32)); 754 } else if (dest->is_double_fpu()) { 755 int index = src->double_stack_ix(); 756 __ fld(dest->as_double_reg(), stack_slot_address(index, c_sz64)); 757 } else { 758 ShouldNotReachHere(); 759 } 760 } 761 762 void LIR_Assembler::klass2reg_with_patching(Register reg, CodeEmitInfo* info) { 763 deoptimize_trap(info); 764 } 765 766 void LIR_Assembler::stack2stack(LIR_Opr src, LIR_Opr dest, BasicType type) { 767 LIR_Opr temp; 768 if (type == T_LONG || type == T_DOUBLE) { 769 temp = FrameMap::t1_long_opr; 770 } else { 771 temp = FrameMap::t1_opr; 772 } 773 774 stack2reg(src, temp, src->type()); 775 reg2stack(temp, dest, dest->type()); 776 } 777 778 void LIR_Assembler::mem2reg(LIR_Opr src, LIR_Opr dest, BasicType type, LIR_PatchCode patch_code, CodeEmitInfo* info, bool wide) { 779 assert(src->is_address(), "should not call otherwise"); 780 assert(dest->is_register(), "should not call otherwise"); 781 782 LIR_Address* addr = src->as_address_ptr(); 783 LIR_Address* from_addr = src->as_address_ptr(); 784 785 if (addr->base()->type() == T_OBJECT) { 786 __ verify_oop(addr->base()->as_pointer_register()); 787 } 788 789 if (patch_code != lir_patch_none) { 790 deoptimize_trap(info); 791 return; 792 } 793 794 if (info != nullptr) { 795 add_debug_info_for_null_check_here(info); 796 } 797 798 int null_check_here = code_offset(); 799 switch (type) { 800 case T_FLOAT: 801 __ flw(dest->as_float_reg(), as_Address(from_addr)); 802 break; 803 case T_DOUBLE: 804 __ fld(dest->as_double_reg(), as_Address(from_addr)); 805 break; 806 case T_ARRAY: // fall through 807 case T_OBJECT: 808 if (UseCompressedOops && !wide) { 809 __ lwu(dest->as_register(), as_Address(from_addr)); 810 } else { 811 __ ld(dest->as_register(), as_Address(from_addr)); 812 } 813 break; 814 case T_METADATA: 815 // We get here to store a method pointer to the stack to pass to 816 // a dtrace runtime call. This can't work on 64 bit with 817 // compressed klass ptrs: T_METADATA can be a compressed klass 818 // ptr or a 64 bit method pointer. 819 ShouldNotReachHere(); 820 __ ld(dest->as_register(), as_Address(from_addr)); 821 break; 822 case T_ADDRESS: 823 __ ld(dest->as_register(), as_Address(from_addr)); 824 break; 825 case T_INT: 826 __ lw(dest->as_register(), as_Address(from_addr)); 827 break; 828 case T_LONG: 829 __ ld(dest->as_register_lo(), as_Address_lo(from_addr)); 830 break; 831 case T_BYTE: 832 __ lb(dest->as_register(), as_Address(from_addr)); 833 break; 834 case T_BOOLEAN: 835 __ lbu(dest->as_register(), as_Address(from_addr)); 836 break; 837 case T_CHAR: 838 __ lhu(dest->as_register(), as_Address(from_addr)); 839 break; 840 case T_SHORT: 841 __ lh(dest->as_register(), as_Address(from_addr)); 842 break; 843 default: 844 ShouldNotReachHere(); 845 } 846 847 if (is_reference_type(type)) { 848 if (UseCompressedOops && !wide) { 849 __ decode_heap_oop(dest->as_register()); 850 } 851 852 __ verify_oop(dest->as_register()); 853 } 854 } 855 856 void LIR_Assembler::move(LIR_Opr src, LIR_Opr dst) { 857 assert(dst->is_cpu_register(), "must be"); 858 assert(dst->type() == src->type(), "must be"); 859 860 if (src->is_cpu_register()) { 861 reg2reg(src, dst); 862 } else if (src->is_stack()) { 863 stack2reg(src, dst, dst->type()); 864 } else if (src->is_constant()) { 865 const2reg(src, dst, lir_patch_none, nullptr); 866 } else { 867 ShouldNotReachHere(); 868 } 869 } 870 871 void LIR_Assembler::emit_op3(LIR_Op3* op) { 872 switch (op->code()) { 873 case lir_idiv: // fall through 874 case lir_irem: 875 arithmetic_idiv(op->code(), 876 op->in_opr1(), 877 op->in_opr2(), 878 op->in_opr3(), 879 op->result_opr(), 880 op->info()); 881 break; 882 case lir_fmad: 883 __ fmadd_d(op->result_opr()->as_double_reg(), 884 op->in_opr1()->as_double_reg(), 885 op->in_opr2()->as_double_reg(), 886 op->in_opr3()->as_double_reg()); 887 break; 888 case lir_fmaf: 889 __ fmadd_s(op->result_opr()->as_float_reg(), 890 op->in_opr1()->as_float_reg(), 891 op->in_opr2()->as_float_reg(), 892 op->in_opr3()->as_float_reg()); 893 break; 894 default: 895 ShouldNotReachHere(); 896 } 897 } 898 899 // Consider using cmov (Zicond) 900 void LIR_Assembler::cmove(LIR_Condition condition, LIR_Opr opr1, LIR_Opr opr2, LIR_Opr result, BasicType type, 901 LIR_Opr cmp_opr1, LIR_Opr cmp_opr2) { 902 Label label; 903 904 emit_branch(condition, cmp_opr1, cmp_opr2, label, /* is_far */ false, 905 /* is_unordered */ (condition == lir_cond_greaterEqual || condition == lir_cond_greater) ? false : true); 906 907 Label done; 908 move_op(opr2, result, type, lir_patch_none, nullptr, 909 false); // wide 910 __ j(done); 911 __ bind(label); 912 move_op(opr1, result, type, lir_patch_none, nullptr, 913 false); // wide 914 __ bind(done); 915 } 916 917 void LIR_Assembler::emit_opBranch(LIR_OpBranch* op) { 918 LIR_Condition condition = op->cond(); 919 if (condition == lir_cond_always) { 920 if (op->info() != nullptr) { 921 add_debug_info_for_branch(op->info()); 922 } 923 } else { 924 assert(op->in_opr1() != LIR_OprFact::illegalOpr && op->in_opr2() != LIR_OprFact::illegalOpr, "conditional branches must have legal operands"); 925 } 926 bool is_unordered = (op->ublock() == op->block()); 927 emit_branch(condition, op->in_opr1(), op->in_opr2(), *op->label(), /* is_far */ true, is_unordered); 928 } 929 930 void LIR_Assembler::emit_branch(LIR_Condition cmp_flag, LIR_Opr cmp1, LIR_Opr cmp2, Label& label, 931 bool is_far, bool is_unordered) { 932 933 if (cmp_flag == lir_cond_always) { 934 __ j(label); 935 return; 936 } 937 938 if (cmp1->is_cpu_register()) { 939 Register reg1 = as_reg(cmp1); 940 if (cmp2->is_cpu_register()) { 941 Register reg2 = as_reg(cmp2); 942 __ c1_cmp_branch(cmp_flag, reg1, reg2, label, cmp1->type(), is_far); 943 } else if (cmp2->is_constant()) { 944 const2reg_helper(cmp2); 945 __ c1_cmp_branch(cmp_flag, reg1, t0, label, cmp2->type(), is_far); 946 } else { 947 ShouldNotReachHere(); 948 } 949 } else if (cmp1->is_single_fpu()) { 950 assert(cmp2->is_single_fpu(), "expect single float register"); 951 __ c1_float_cmp_branch(cmp_flag, cmp1->as_float_reg(), cmp2->as_float_reg(), label, is_far, is_unordered); 952 } else if (cmp1->is_double_fpu()) { 953 assert(cmp2->is_double_fpu(), "expect double float register"); 954 __ c1_float_cmp_branch(cmp_flag | C1_MacroAssembler::c1_double_branch_mask, 955 cmp1->as_double_reg(), cmp2->as_double_reg(), label, is_far, is_unordered); 956 } else { 957 ShouldNotReachHere(); 958 } 959 } 960 961 void LIR_Assembler::emit_opConvert(LIR_OpConvert* op) { 962 LIR_Opr src = op->in_opr(); 963 LIR_Opr dest = op->result_opr(); 964 965 switch (op->bytecode()) { 966 case Bytecodes::_i2f: 967 __ fcvt_s_w(dest->as_float_reg(), src->as_register()); break; 968 case Bytecodes::_i2d: 969 __ fcvt_d_w(dest->as_double_reg(), src->as_register()); break; 970 case Bytecodes::_l2d: 971 __ fcvt_d_l(dest->as_double_reg(), src->as_register_lo()); break; 972 case Bytecodes::_l2f: 973 __ fcvt_s_l(dest->as_float_reg(), src->as_register_lo()); break; 974 case Bytecodes::_f2d: 975 __ fcvt_d_s(dest->as_double_reg(), src->as_float_reg()); break; 976 case Bytecodes::_d2f: 977 __ fcvt_s_d(dest->as_float_reg(), src->as_double_reg()); break; 978 case Bytecodes::_i2c: 979 __ zext(dest->as_register(), src->as_register(), 16); break; 980 case Bytecodes::_i2l: 981 __ sext(dest->as_register_lo(), src->as_register(), 32); break; 982 case Bytecodes::_i2s: 983 __ sext(dest->as_register(), src->as_register(), 16); break; 984 case Bytecodes::_i2b: 985 __ sext(dest->as_register(), src->as_register(), 8); break; 986 case Bytecodes::_l2i: 987 __ sext(dest->as_register(), src->as_register_lo(), 32); break; 988 case Bytecodes::_d2l: 989 __ fcvt_l_d_safe(dest->as_register_lo(), src->as_double_reg()); break; 990 case Bytecodes::_f2i: 991 __ fcvt_w_s_safe(dest->as_register(), src->as_float_reg()); break; 992 case Bytecodes::_f2l: 993 __ fcvt_l_s_safe(dest->as_register_lo(), src->as_float_reg()); break; 994 case Bytecodes::_d2i: 995 __ fcvt_w_d_safe(dest->as_register(), src->as_double_reg()); break; 996 default: 997 ShouldNotReachHere(); 998 } 999 } 1000 1001 void LIR_Assembler::emit_alloc_obj(LIR_OpAllocObj* op) { 1002 if (op->init_check()) { 1003 __ lbu(t0, Address(op->klass()->as_register(), 1004 InstanceKlass::init_state_offset())); 1005 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); 1006 __ mv(t1, (u1)InstanceKlass::fully_initialized); 1007 add_debug_info_for_null_check_here(op->stub()->info()); 1008 __ bne(t0, t1, *op->stub()->entry(), /* is_far */ true); 1009 } 1010 1011 __ allocate_object(op->obj()->as_register(), 1012 op->tmp1()->as_register(), 1013 op->tmp2()->as_register(), 1014 op->header_size(), 1015 op->object_size(), 1016 op->klass()->as_register(), 1017 *op->stub()->entry()); 1018 1019 __ bind(*op->stub()->continuation()); 1020 } 1021 1022 void LIR_Assembler::emit_alloc_array(LIR_OpAllocArray* op) { 1023 Register len = op->len()->as_register(); 1024 1025 if (UseSlowPath || op->always_slow_path() || 1026 (!UseFastNewObjectArray && is_reference_type(op->type())) || 1027 (!UseFastNewTypeArray && !is_reference_type(op->type()))) { 1028 __ j(*op->stub()->entry()); 1029 } else { 1030 Register tmp1 = op->tmp1()->as_register(); 1031 Register tmp2 = op->tmp2()->as_register(); 1032 Register tmp3 = op->tmp3()->as_register(); 1033 if (len == tmp1) { 1034 tmp1 = tmp3; 1035 } else if (len == tmp2) { 1036 tmp2 = tmp3; 1037 } else if (len == tmp3) { 1038 // everything is ok 1039 } else { 1040 __ mv(tmp3, len); 1041 } 1042 __ allocate_array(op->obj()->as_register(), 1043 len, 1044 tmp1, 1045 tmp2, 1046 arrayOopDesc::base_offset_in_bytes(op->type()), 1047 array_element_size(op->type()), 1048 op->klass()->as_register(), 1049 *op->stub()->entry(), 1050 op->zero_array()); 1051 } 1052 __ bind(*op->stub()->continuation()); 1053 } 1054 1055 void LIR_Assembler::type_profile_helper(Register mdo, ciMethodData *md, 1056 ciProfileData *data, Register recv) { 1057 int mdp_offset = md->byte_offset_of_slot(data, in_ByteSize(0)); 1058 __ profile_receiver_type(recv, mdo, mdp_offset); 1059 } 1060 1061 void LIR_Assembler::data_check(LIR_OpTypeCheck *op, ciMethodData **md, ciProfileData **data) { 1062 ciMethod* method = op->profiled_method(); 1063 assert(method != nullptr, "Should have method"); 1064 int bci = op->profiled_bci(); 1065 *md = method->method_data_or_null(); 1066 guarantee(*md != nullptr, "Sanity"); 1067 *data = ((*md)->bci_to_data(bci)); 1068 assert(*data != nullptr, "need data for type check"); 1069 assert((*data)->is_ReceiverTypeData(), "need ReceiverTypeData for type check"); 1070 } 1071 1072 void LIR_Assembler::typecheck_helper_slowcheck(ciKlass *k, Register obj, Register Rtmp1, 1073 Register k_RInfo, Register klass_RInfo, 1074 Label *failure_target, Label *success_target) { 1075 // get object class 1076 // not a safepoint as obj null check happens earlier 1077 __ load_klass(klass_RInfo, obj); 1078 if (k->is_loaded()) { 1079 // See if we get an immediate positive hit 1080 __ ld(t0, Address(klass_RInfo, int64_t(k->super_check_offset()))); 1081 if ((juint)in_bytes(Klass::secondary_super_cache_offset()) != k->super_check_offset()) { 1082 __ bne(k_RInfo, t0, *failure_target, /* is_far */ true); 1083 // successful cast, fall through to profile or jump 1084 } else { 1085 // See if we get an immediate positive hit 1086 __ beq(k_RInfo, t0, *success_target); 1087 // check for self 1088 if (k->is_loaded() && k->is_obj_array_klass()) { 1089 // For a direct pointer comparison, we need the refined array klass pointer 1090 ciKlass* k_refined = ciObjArrayKlass::make(k->as_obj_array_klass()->element_klass()); 1091 if (!k_refined->is_loaded()) { 1092 bailout("encountered unloaded_ciobjarrayklass due to out of memory error"); 1093 return; 1094 } 1095 __ mov_metadata(t0, k_refined->constant_encoding()); 1096 __ beq(klass_RInfo, t0, *success_target); 1097 } else { 1098 __ beq(klass_RInfo, k_RInfo, *success_target); 1099 } 1100 __ subi(sp, sp, 2 * wordSize); // 2: store k_RInfo and klass_RInfo 1101 __ sd(k_RInfo, Address(sp, 0)); // sub klass 1102 __ sd(klass_RInfo, Address(sp, wordSize)); // super klass 1103 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id))); 1104 // load result to k_RInfo 1105 __ ld(k_RInfo, Address(sp, 0)); 1106 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo 1107 // result is a boolean 1108 __ beqz(k_RInfo, *failure_target, /* is_far */ true); 1109 // successful cast, fall through to profile or jump 1110 } 1111 } else { 1112 // perform the fast part of the checking logic 1113 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr); 1114 // call out-of-line instance of __ check_klass_subtytpe_slow_path(...) 1115 __ subi(sp, sp, 2 * wordSize); // 2: store k_RInfo and klass_RInfo 1116 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass 1117 __ sd(k_RInfo, Address(sp, 0)); // super klass 1118 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id))); 1119 // load result to k_RInfo 1120 __ ld(k_RInfo, Address(sp, 0)); 1121 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo 1122 // result is a boolean 1123 __ beqz(k_RInfo, *failure_target, /* is_far */ true); 1124 // successful cast, fall thriugh to profile or jump 1125 } 1126 } 1127 1128 void LIR_Assembler::profile_object(ciMethodData* md, ciProfileData* data, Register obj, 1129 Register k_RInfo, Register klass_RInfo, Label* obj_is_null) { 1130 Register mdo = klass_RInfo; 1131 __ mov_metadata(mdo, md->constant_encoding()); 1132 Label not_null; 1133 __ bnez(obj, not_null); 1134 // Object is null, update MDO and exit 1135 Address data_addr = __ form_address(t1, mdo, md->byte_offset_of_slot(data, DataLayout::flags_offset())); 1136 __ lbu(t0, data_addr); 1137 __ ori(t0, t0, BitData::null_seen_byte_constant()); 1138 __ sb(t0, data_addr); 1139 __ j(*obj_is_null); 1140 __ bind(not_null); 1141 1142 Register recv = k_RInfo; 1143 __ load_klass(recv, obj); 1144 type_profile_helper(mdo, md, data, recv); 1145 } 1146 1147 void LIR_Assembler::typecheck_loaded(LIR_OpTypeCheck *op, ciKlass* k, Register k_RInfo) { 1148 if (!k->is_loaded()) { 1149 klass2reg_with_patching(k_RInfo, op->info_for_patch()); 1150 } else { 1151 __ mov_metadata(k_RInfo, k->constant_encoding()); 1152 } 1153 } 1154 1155 void LIR_Assembler::emit_typecheck_helper(LIR_OpTypeCheck *op, Label* success, Label* failure, Label* obj_is_null) { 1156 Register obj = op->object()->as_register(); 1157 Register k_RInfo = op->tmp1()->as_register(); 1158 Register klass_RInfo = op->tmp2()->as_register(); 1159 Register dst = op->result_opr()->as_register(); 1160 ciKlass* k = op->klass(); 1161 Register Rtmp1 = noreg; 1162 1163 // check if it needs to be profiled 1164 ciMethodData* md = nullptr; 1165 ciProfileData* data = nullptr; 1166 1167 const bool should_profile = op->should_profile(); 1168 if (should_profile) { 1169 data_check(op, &md, &data); 1170 } 1171 Label* success_target = success; 1172 Label* failure_target = failure; 1173 1174 if (obj == k_RInfo) { 1175 k_RInfo = dst; 1176 } else if (obj == klass_RInfo) { 1177 klass_RInfo = dst; 1178 } 1179 Rtmp1 = op->tmp3()->as_register(); 1180 select_different_registers(obj, dst, k_RInfo, klass_RInfo, Rtmp1); 1181 1182 assert_different_registers(obj, k_RInfo, klass_RInfo); 1183 1184 if (op->need_null_check()) { 1185 if (should_profile) { 1186 profile_object(md, data, obj, k_RInfo, klass_RInfo, obj_is_null); 1187 } else { 1188 __ beqz(obj, *obj_is_null); 1189 } 1190 } 1191 1192 typecheck_loaded(op, k, k_RInfo); 1193 __ verify_oop(obj); 1194 1195 if (op->fast_check()) { 1196 assert(!k->is_loaded() || !k->is_obj_array_klass(), "Use refined array for a direct pointer comparison"); 1197 // get object class 1198 // not a safepoint as obj null check happens earlier 1199 __ load_klass(t0, obj, t1); 1200 __ bne(t0, k_RInfo, *failure_target, /* is_far */ true); 1201 // successful cast, fall through to profile or jump 1202 } else { 1203 typecheck_helper_slowcheck(k, obj, Rtmp1, k_RInfo, klass_RInfo, failure_target, success_target); 1204 } 1205 1206 __ j(*success); 1207 } 1208 1209 void LIR_Assembler::emit_opTypeCheck(LIR_OpTypeCheck* op) { 1210 const bool should_profile = op->should_profile(); 1211 1212 LIR_Code code = op->code(); 1213 if (code == lir_store_check) { 1214 typecheck_lir_store(op, should_profile); 1215 } else if (code == lir_checkcast) { 1216 Register obj = op->object()->as_register(); 1217 Register dst = op->result_opr()->as_register(); 1218 Label success; 1219 emit_typecheck_helper(op, &success, op->stub()->entry(), &success); 1220 __ bind(success); 1221 if (dst != obj) { 1222 __ mv(dst, obj); 1223 } 1224 } else if (code == lir_instanceof) { 1225 Register obj = op->object()->as_register(); 1226 Register dst = op->result_opr()->as_register(); 1227 Label success, failure, done; 1228 emit_typecheck_helper(op, &success, &failure, &failure); 1229 __ bind(failure); 1230 __ mv(dst, zr); 1231 __ j(done); 1232 __ bind(success); 1233 __ mv(dst, 1); 1234 __ bind(done); 1235 } else { 1236 ShouldNotReachHere(); 1237 } 1238 } 1239 1240 void LIR_Assembler::emit_opFlattenedArrayCheck(LIR_OpFlattenedArrayCheck* op) { 1241 // We are loading/storing from/to an array that *may* be a flat array (the 1242 // declared type is Object[], abstract[], interface[] or VT.ref[]). 1243 // If this array is a flat array, take the slow path. 1244 __ test_flat_array_oop(op->array()->as_register(), op->tmp()->as_register(), *op->stub()->entry()); 1245 } 1246 1247 void LIR_Assembler::emit_opNullFreeArrayCheck(LIR_OpNullFreeArrayCheck* op) { 1248 // We are storing into an array that *may* be null-free (the declared type is 1249 // Object[], abstract[], interface[] or VT.ref[]). 1250 Label test_mark_word; 1251 Register tmp = op->tmp()->as_register(); 1252 __ ld(tmp, Address(op->array()->as_register(), oopDesc::mark_offset_in_bytes())); 1253 __ test_bit(t0, tmp, exact_log2(markWord::unlocked_value)); 1254 __ bnez(t0, test_mark_word); 1255 __ load_prototype_header(tmp, op->array()->as_register()); 1256 __ bind(test_mark_word); 1257 __ test_bit(tmp, tmp, exact_log2(markWord::null_free_array_bit_in_place)); 1258 } 1259 1260 void LIR_Assembler::emit_opSubstitutabilityCheck(LIR_OpSubstitutabilityCheck* op) { 1261 Label L_oops_equal; 1262 Label L_oops_not_equal; 1263 Label L_end; 1264 1265 Register left = op->left()->as_register(); 1266 Register right = op->right()->as_register(); 1267 1268 __ beq(left, right, L_oops_equal); 1269 1270 // (1) Null check -- if one of the operands is null, the other must not be null (because 1271 // the two references are not equal), so they are not substitutable, 1272 __ beqz(left, L_oops_not_equal); 1273 __ beqz(right, L_oops_not_equal); 1274 1275 ciKlass* left_klass = op->left_klass(); 1276 ciKlass* right_klass = op->right_klass(); 1277 1278 // (2) Inline type check -- if either of the operands is not a inline type, 1279 // they are not substitutable. We do this only if we are not sure that the 1280 // operands are inline type 1281 if ((left_klass == nullptr || right_klass == nullptr) ||// The klass is still unloaded, or came from a Phi node. 1282 !left_klass->is_inlinetype() || !right_klass->is_inlinetype()) { 1283 Register tmp1 = op->tmp1()->as_register(); 1284 Register tmp2 = op->tmp2()->as_register(); 1285 __ mv(tmp1, markWord::inline_type_pattern); 1286 __ ld(tmp2, Address(left, oopDesc::mark_offset_in_bytes())); 1287 __ andr(tmp1, tmp1, tmp2); 1288 __ ld(tmp2, Address(right, oopDesc::mark_offset_in_bytes())); 1289 __ andr(tmp1, tmp1, tmp2); 1290 __ mv(tmp2, (u1)markWord::inline_type_pattern); 1291 __ bne(tmp1, tmp2, L_oops_not_equal); 1292 } 1293 1294 // (3) Same klass check: if the operands are of different klasses, they are not substitutable. 1295 if (left_klass != nullptr && left_klass->is_inlinetype() && left_klass == right_klass) { 1296 // No need to load klass -- the operands are statically known to be the same inline klass. 1297 __ j(*op->stub()->entry()); 1298 } else { 1299 Register left_klass_op = op->tmp1()->as_register(); 1300 Register right_klass_op = op->tmp2()->as_register(); 1301 if (UseCompactObjectHeaders) { 1302 __ load_narrow_klass_compact(left_klass_op, left); 1303 __ load_narrow_klass_compact(right_klass_op, right); 1304 } else { 1305 __ lwu(left_klass_op, Address(left, oopDesc::klass_offset_in_bytes())); 1306 __ lwu(right_klass_op, Address(right, oopDesc::klass_offset_in_bytes())); 1307 } 1308 __ beq(left_klass_op, right_klass_op, *op->stub()->entry(), /* is_far */ true); // same klass -> do slow check 1309 // fall through to L_oops_not_equal 1310 } 1311 1312 __ bind(L_oops_not_equal); 1313 move(op->not_equal_result(), op->result_opr()); 1314 __ j(L_end); 1315 1316 // We've returned from the stub. X10 contains 0x0 IFF the two 1317 // operands are not substitutable. (Don't compare against 0x1 in case the 1318 // C compiler is naughty) 1319 __ bind(*op->stub()->continuation()); 1320 __ beqz(x10, L_oops_not_equal); // (call_stub() == 0x0) -> not_equal 1321 1322 __ bind(L_oops_equal); 1323 move(op->equal_result(), op->result_opr()); // (call_stub() != 0x0) -> equal 1324 // fall-through 1325 __ bind(L_end); 1326 } 1327 1328 void LIR_Assembler::emit_profile_inline_type(LIR_OpProfileInlineType* op) { 1329 Register obj = op->obj()->as_register(); 1330 Register tmp = op->tmp()->as_pointer_register(); 1331 bool not_null = op->not_null(); 1332 int flag = op->flag(); 1333 1334 assert_different_registers(tmp, t0, t1); 1335 1336 Label not_inline_type; 1337 if (!not_null) { 1338 __ beqz(obj, not_inline_type); 1339 } 1340 1341 __ test_oop_is_not_inline_type(obj, tmp, not_inline_type); 1342 1343 Address mdo_addr = as_Address(op->mdp()->as_address_ptr(), t1); 1344 __ lbu(tmp, mdo_addr); 1345 __ mv(t0, flag); 1346 __ orr(tmp, tmp, t0); 1347 __ sb(tmp, mdo_addr); 1348 1349 __ bind(not_inline_type); 1350 } 1351 1352 void LIR_Assembler::check_orig_pc() { 1353 Unimplemented(); 1354 } 1355 1356 void LIR_Assembler::emit_compare_and_swap(LIR_OpCompareAndSwap* op) { 1357 Register addr; 1358 if (op->addr()->is_register()) { 1359 addr = as_reg(op->addr()); 1360 } else { 1361 assert(op->addr()->is_address(), "what else?"); 1362 LIR_Address* addr_ptr = op->addr()->as_address_ptr(); 1363 assert(addr_ptr->disp() == 0, "need 0 disp"); 1364 assert(addr_ptr->index() == LIR_Opr::illegalOpr(), "need 0 index"); 1365 addr = as_reg(addr_ptr->base()); 1366 } 1367 Register newval = as_reg(op->new_value()); 1368 Register cmpval = as_reg(op->cmp_value()); 1369 1370 if (op->code() == lir_cas_obj) { 1371 if (UseCompressedOops) { 1372 Register tmp1 = op->tmp1()->as_register(); 1373 assert(op->tmp1()->is_valid(), "must be"); 1374 Register tmp2 = op->tmp2()->as_register(); 1375 assert(op->tmp2()->is_valid(), "must be"); 1376 1377 __ encode_heap_oop(tmp1, cmpval); 1378 cmpval = tmp1; 1379 __ encode_heap_oop(tmp2, newval); 1380 newval = tmp2; 1381 caswu(addr, newval, cmpval); 1382 } else { 1383 casl(addr, newval, cmpval); 1384 } 1385 } else if (op->code() == lir_cas_int) { 1386 casw(addr, newval, cmpval); 1387 } else { 1388 casl(addr, newval, cmpval); 1389 } 1390 1391 if (op->result_opr()->is_valid()) { 1392 assert(op->result_opr()->is_register(), "need a register"); 1393 __ mv(as_reg(op->result_opr()), t0); // cas result in t0, and 0 for success 1394 } 1395 } 1396 1397 void LIR_Assembler::intrinsic_op(LIR_Code code, LIR_Opr value, LIR_Opr unused, LIR_Opr dest, LIR_Op* op) { 1398 switch (code) { 1399 case lir_abs: __ fabs_d(dest->as_double_reg(), value->as_double_reg()); break; 1400 case lir_sqrt: __ fsqrt_d(dest->as_double_reg(), value->as_double_reg()); break; 1401 default: ShouldNotReachHere(); 1402 } 1403 } 1404 1405 void LIR_Assembler::logic_op(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst) { 1406 assert(left->is_single_cpu() || left->is_double_cpu(), "expect single or double register"); 1407 Register Rleft = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 1408 if (dst->is_single_cpu()) { 1409 Register Rdst = dst->as_register(); 1410 if (right->is_constant()) { 1411 int right_const = right->as_jint(); 1412 if (Assembler::is_simm12(right_const)) { 1413 logic_op_imm(Rdst, Rleft, right_const, code); 1414 __ sext(Rdst, Rdst, 32); 1415 } else { 1416 __ mv(t0, right_const); 1417 logic_op_reg32(Rdst, Rleft, t0, code); 1418 } 1419 } else { 1420 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo(); 1421 logic_op_reg32(Rdst, Rleft, Rright, code); 1422 } 1423 } else { 1424 Register Rdst = dst->as_register_lo(); 1425 if (right->is_constant()) { 1426 long right_const = right->as_jlong(); 1427 if (Assembler::is_simm12(right_const)) { 1428 logic_op_imm(Rdst, Rleft, right_const, code); 1429 } else { 1430 __ mv(t0, right_const); 1431 logic_op_reg(Rdst, Rleft, t0, code); 1432 } 1433 } else { 1434 Register Rright = right->is_single_cpu() ? right->as_register() : right->as_register_lo(); 1435 logic_op_reg(Rdst, Rleft, Rright, code); 1436 } 1437 } 1438 } 1439 1440 void LIR_Assembler::comp_op(LIR_Condition condition, LIR_Opr src, LIR_Opr result, LIR_Op2* op) { 1441 ShouldNotCallThis(); 1442 } 1443 1444 void LIR_Assembler::comp_fl2i(LIR_Code code, LIR_Opr left, LIR_Opr right, LIR_Opr dst, LIR_Op2* op) { 1445 if (code == lir_cmp_fd2i || code == lir_ucmp_fd2i) { 1446 bool is_unordered_less = (code == lir_ucmp_fd2i); 1447 if (left->is_single_fpu()) { 1448 __ float_cmp(true, is_unordered_less ? -1 : 1, 1449 left->as_float_reg(), right->as_float_reg(), dst->as_register()); 1450 } else if (left->is_double_fpu()) { 1451 __ float_cmp(false, is_unordered_less ? -1 : 1, 1452 left->as_double_reg(), right->as_double_reg(), dst->as_register()); 1453 } else { 1454 ShouldNotReachHere(); 1455 } 1456 } else if (code == lir_cmp_l2i) { 1457 __ cmp_l2i(dst->as_register(), left->as_register_lo(), right->as_register_lo()); 1458 } else { 1459 ShouldNotReachHere(); 1460 } 1461 } 1462 1463 void LIR_Assembler::align_call(LIR_Code code) { 1464 // With RVC a call instruction may get 2-byte aligned. 1465 // The address of the call instruction needs to be 4-byte aligned to 1466 // ensure that it does not span a cache line so that it can be patched. 1467 __ align(NativeInstruction::instruction_size); 1468 } 1469 1470 void LIR_Assembler::call(LIR_OpJavaCall* op, relocInfo::relocType rtype) { 1471 Assembler::IncompressibleScope scope(_masm); 1472 address call = __ reloc_call(Address(op->addr(), rtype)); 1473 if (call == nullptr) { 1474 bailout("reloc call address stub overflow"); 1475 return; 1476 } 1477 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields()); 1478 __ post_call_nop(); 1479 } 1480 1481 void LIR_Assembler::ic_call(LIR_OpJavaCall* op) { 1482 Assembler::IncompressibleScope scope(_masm); 1483 address call = __ ic_call(op->addr()); 1484 if (call == nullptr) { 1485 bailout("reloc call address stub overflow"); 1486 return; 1487 } 1488 add_call_info(code_offset(), op->info(), op->maybe_return_as_fields()); 1489 __ post_call_nop(); 1490 } 1491 1492 void LIR_Assembler::emit_static_call_stub() { 1493 address call_pc = __ pc(); 1494 MacroAssembler::assert_alignment(call_pc); 1495 address stub = __ start_a_stub(call_stub_size()); 1496 if (stub == nullptr) { 1497 bailout("static call stub overflow"); 1498 return; 1499 } 1500 1501 int start = __ offset(); 1502 1503 __ relocate(static_stub_Relocation::spec(call_pc)); 1504 __ emit_static_call_stub(); 1505 1506 assert(__ offset() - start + CompiledDirectCall::to_trampoline_stub_size() 1507 <= call_stub_size(), "stub too big"); 1508 __ end_a_stub(); 1509 } 1510 1511 void LIR_Assembler::throw_op(LIR_Opr exceptionPC, LIR_Opr exceptionOop, CodeEmitInfo* info) { 1512 assert(exceptionOop->as_register() == x10, "must match"); 1513 assert(exceptionPC->as_register() == x13, "must match"); 1514 1515 // exception object is not added to oop map by LinearScan 1516 // (LinearScan assumes that no oops are in fixed registers) 1517 info->add_register_oop(exceptionOop); 1518 StubId unwind_id; 1519 1520 // get current pc information 1521 // pc is only needed if the method has an exception handler, the unwind code does not need it. 1522 if (compilation()->debug_info_recorder()->last_pc_offset() == __ offset()) { 1523 // As no instructions have been generated yet for this LIR node it's 1524 // possible that an oop map already exists for the current offset. 1525 // In that case insert an dummy NOP here to ensure all oop map PCs 1526 // are unique. See JDK-8237483. 1527 __ nop(); 1528 } 1529 int pc_for_athrow_offset = __ offset(); 1530 InternalAddress pc_for_athrow(__ pc()); 1531 __ la(exceptionPC->as_register(), pc_for_athrow); 1532 add_call_info(pc_for_athrow_offset, info); // for exception handler 1533 1534 __ verify_not_null_oop(x10); 1535 // search an exception handler (x10: exception oop, x13: throwing pc) 1536 if (compilation()->has_fpu_code()) { 1537 unwind_id = StubId::c1_handle_exception_id; 1538 } else { 1539 unwind_id = StubId::c1_handle_exception_nofpu_id; 1540 } 1541 __ far_call(RuntimeAddress(Runtime1::entry_for(unwind_id))); 1542 __ nop(); 1543 } 1544 1545 void LIR_Assembler::unwind_op(LIR_Opr exceptionOop) { 1546 assert(exceptionOop->as_register() == x10, "must match"); 1547 __ j(_unwind_handler_entry); 1548 } 1549 1550 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, LIR_Opr count, LIR_Opr dest, LIR_Opr tmp) { 1551 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 1552 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 1553 Register count_reg = count->as_register(); 1554 if (dest->is_single_cpu()) { 1555 assert (dest->type() == T_INT, "unexpected result type"); 1556 assert (left->type() == T_INT, "unexpected left type"); 1557 __ andi(t0, count_reg, 31); // should not shift more than 31 bits 1558 switch (code) { 1559 case lir_shl: __ sllw(dest_reg, left_reg, t0); break; 1560 case lir_shr: __ sraw(dest_reg, left_reg, t0); break; 1561 case lir_ushr: __ srlw(dest_reg, left_reg, t0); break; 1562 default: ShouldNotReachHere(); 1563 } 1564 } else if (dest->is_double_cpu()) { 1565 __ andi(t0, count_reg, 63); // should not shift more than 63 bits 1566 switch (code) { 1567 case lir_shl: __ sll(dest_reg, left_reg, t0); break; 1568 case lir_shr: __ sra(dest_reg, left_reg, t0); break; 1569 case lir_ushr: __ srl(dest_reg, left_reg, t0); break; 1570 default: ShouldNotReachHere(); 1571 } 1572 } else { 1573 ShouldNotReachHere(); 1574 } 1575 } 1576 1577 void LIR_Assembler::shift_op(LIR_Code code, LIR_Opr left, jint count, LIR_Opr dest) { 1578 Register left_reg = left->is_single_cpu() ? left->as_register() : left->as_register_lo(); 1579 Register dest_reg = dest->is_single_cpu() ? dest->as_register() : dest->as_register_lo(); 1580 if (dest->is_single_cpu()) { 1581 assert (dest->type() == T_INT, "unexpected result type"); 1582 assert (left->type() == T_INT, "unexpected left type"); 1583 count &= 0x1f; 1584 if (count != 0) { 1585 switch (code) { 1586 case lir_shl: __ slliw(dest_reg, left_reg, count); break; 1587 case lir_shr: __ sraiw(dest_reg, left_reg, count); break; 1588 case lir_ushr: __ srliw(dest_reg, left_reg, count); break; 1589 default: ShouldNotReachHere(); 1590 } 1591 } else { 1592 move_regs(left_reg, dest_reg); 1593 } 1594 } else if (dest->is_double_cpu()) { 1595 count &= 0x3f; 1596 if (count != 0) { 1597 switch (code) { 1598 case lir_shl: __ slli(dest_reg, left_reg, count); break; 1599 case lir_shr: __ srai(dest_reg, left_reg, count); break; 1600 case lir_ushr: __ srli(dest_reg, left_reg, count); break; 1601 default: ShouldNotReachHere(); 1602 } 1603 } else { 1604 move_regs(left->as_register_lo(), dest->as_register_lo()); 1605 } 1606 } else { 1607 ShouldNotReachHere(); 1608 } 1609 } 1610 1611 void LIR_Assembler::emit_lock(LIR_OpLock* op) { 1612 Register obj = op->obj_opr()->as_register(); // may not be an oop 1613 Register hdr = op->hdr_opr()->as_register(); 1614 Register lock = op->lock_opr()->as_register(); 1615 Register temp = op->scratch_opr()->as_register(); 1616 if (op->code() == lir_lock) { 1617 // add debug info for NullPointerException only if one is possible 1618 int null_check_offset = __ lock_object(hdr, obj, lock, temp, *op->stub()->entry()); 1619 if (op->info() != nullptr) { 1620 add_debug_info_for_null_check(null_check_offset, op->info()); 1621 } 1622 } else if (op->code() == lir_unlock) { 1623 __ unlock_object(hdr, obj, lock, temp, *op->stub()->entry()); 1624 } else { 1625 Unimplemented(); 1626 } 1627 __ bind(*op->stub()->continuation()); 1628 } 1629 1630 void LIR_Assembler::emit_load_klass(LIR_OpLoadKlass* op) { 1631 Register obj = op->obj()->as_pointer_register(); 1632 Register result = op->result_opr()->as_pointer_register(); 1633 1634 CodeEmitInfo* info = op->info(); 1635 if (info != nullptr) { 1636 add_debug_info_for_null_check_here(info); 1637 } 1638 1639 __ load_klass(result, obj); 1640 } 1641 1642 void LIR_Assembler::emit_profile_call(LIR_OpProfileCall* op) { 1643 ciMethod* method = op->profiled_method(); 1644 int bci = op->profiled_bci(); 1645 1646 // Update counter for all call types 1647 ciMethodData* md = method->method_data_or_null(); 1648 guarantee(md != nullptr, "Sanity"); 1649 ciProfileData* data = md->bci_to_data(bci); 1650 assert(data != nullptr && data->is_CounterData(), "need CounterData for calls"); 1651 assert(op->mdo()->is_single_cpu(), "mdo must be allocated"); 1652 Register mdo = op->mdo()->as_register(); 1653 __ mov_metadata(mdo, md->constant_encoding()); 1654 Address counter_addr(mdo, md->byte_offset_of_slot(data, CounterData::count_offset())); 1655 // Perform additional virtual call profiling for invokevirtual and 1656 // invokeinterface bytecodes 1657 if (op->should_profile_receiver_type()) { 1658 assert(op->recv()->is_single_cpu(), "recv must be allocated"); 1659 Register recv = op->recv()->as_register(); 1660 assert_different_registers(mdo, recv); 1661 assert(data->is_VirtualCallData(), "need VirtualCallData for virtual calls"); 1662 ciKlass* known_klass = op->known_holder(); 1663 if (C1OptimizeVirtualCallProfiling && known_klass != nullptr) { 1664 // We know the type that will be seen at this call site; we can 1665 // statically update the MethodData* rather than needing to do 1666 // dynamic tests on the receiver type 1667 ciVirtualCallData* vc_data = (ciVirtualCallData*) data; 1668 for (uint i = 0; i < VirtualCallData::row_limit(); i++) { 1669 ciKlass* receiver = vc_data->receiver(i); 1670 if (known_klass->equals(receiver)) { 1671 Address data_addr(mdo, md->byte_offset_of_slot(data, VirtualCallData::receiver_count_offset(i))); 1672 __ increment(data_addr, DataLayout::counter_increment); 1673 return; 1674 } 1675 } 1676 // Receiver type is not found in profile data. 1677 // Fall back to runtime helper to handle the rest at runtime. 1678 __ mov_metadata(recv, known_klass->constant_encoding()); 1679 } else { 1680 __ load_klass(recv, recv); 1681 } 1682 type_profile_helper(mdo, md, data, recv); 1683 } else { 1684 // Static call 1685 __ increment(counter_addr, DataLayout::counter_increment); 1686 } 1687 } 1688 1689 void LIR_Assembler::monitor_address(int monitor_no, LIR_Opr dst) { 1690 __ la(dst->as_register(), frame_map()->address_for_monitor_lock(monitor_no)); 1691 } 1692 1693 void LIR_Assembler::emit_updatecrc32(LIR_OpUpdateCRC32* op) { 1694 assert(op->crc()->is_single_cpu(), "crc must be register"); 1695 assert(op->val()->is_single_cpu(), "byte value must be register"); 1696 assert(op->result_opr()->is_single_cpu(), "result must be register"); 1697 Register crc = op->crc()->as_register(); 1698 Register val = op->val()->as_register(); 1699 Register res = op->result_opr()->as_register(); 1700 1701 assert_different_registers(val, crc, res); 1702 __ la(res, ExternalAddress(StubRoutines::crc_table_addr())); 1703 1704 __ notr(crc, crc); // ~crc 1705 __ zext(crc, crc, 32); 1706 __ update_byte_crc32(crc, val, res); 1707 __ notr(res, crc); // ~crc 1708 } 1709 1710 void LIR_Assembler::check_conflict(ciKlass* exact_klass, intptr_t current_klass, 1711 Register tmp, Label &next, Label &none, 1712 Address mdo_addr) { 1713 if (exact_klass == nullptr || TypeEntries::is_type_none(current_klass)) { 1714 if (exact_klass != nullptr) { 1715 __ mov_metadata(tmp, exact_klass->constant_encoding()); 1716 } else { 1717 __ load_klass(tmp, tmp); 1718 } 1719 1720 __ ld(t1, mdo_addr); 1721 __ xorr(tmp, tmp, t1); 1722 __ andi(t0, tmp, TypeEntries::type_klass_mask); 1723 // klass seen before, nothing to do. The unknown bit may have been 1724 // set already but no need to check. 1725 __ beqz(t0, next); 1726 1727 // already unknown. Nothing to do anymore. 1728 __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown)); 1729 __ bnez(t0, next); 1730 1731 if (TypeEntries::is_type_none(current_klass)) { 1732 __ beqz(t1, none); 1733 __ mv(t0, (u1)TypeEntries::null_seen); 1734 __ beq(t0, t1, none); 1735 // There is a chance that the checks above 1736 // fail if another thread has just set the 1737 // profiling to this obj's klass 1738 __ membar(MacroAssembler::LoadLoad); 1739 __ xorr(tmp, tmp, t1); // get back original value before XOR 1740 __ ld(t1, mdo_addr); 1741 __ xorr(tmp, tmp, t1); 1742 __ andi(t0, tmp, TypeEntries::type_klass_mask); 1743 __ beqz(t0, next); 1744 } 1745 } else { 1746 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr && 1747 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "conflict only"); 1748 1749 __ ld(tmp, mdo_addr); 1750 // already unknown. Nothing to do anymore. 1751 __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown)); 1752 __ bnez(t0, next); 1753 } 1754 1755 // different than before. Cannot keep accurate profile. 1756 __ ld(t1, mdo_addr); 1757 __ ori(t1, t1, TypeEntries::type_unknown); 1758 __ sd(t1, mdo_addr); 1759 1760 if (TypeEntries::is_type_none(current_klass)) { 1761 __ j(next); 1762 1763 __ bind(none); 1764 // first time here. Set profile type. 1765 __ sd(tmp, mdo_addr); 1766 #ifdef ASSERT 1767 __ andi(tmp, tmp, TypeEntries::type_mask); 1768 __ verify_klass_ptr(tmp); 1769 #endif 1770 } 1771 } 1772 1773 void LIR_Assembler::check_no_conflict(ciKlass* exact_klass, intptr_t current_klass, Register tmp, 1774 Address mdo_addr, Label &next) { 1775 // There's a single possible klass at this profile point 1776 assert(exact_klass != nullptr, "should be"); 1777 if (TypeEntries::is_type_none(current_klass)) { 1778 __ mov_metadata(tmp, exact_klass->constant_encoding()); 1779 __ ld(t1, mdo_addr); 1780 __ xorr(tmp, tmp, t1); 1781 __ andi(t0, tmp, TypeEntries::type_klass_mask); 1782 __ beqz(t0, next); 1783 #ifdef ASSERT 1784 { 1785 Label ok; 1786 __ ld(t0, mdo_addr); 1787 __ beqz(t0, ok); 1788 __ mv(t1, (u1)TypeEntries::null_seen); 1789 __ beq(t0, t1, ok); 1790 // may have been set by another thread 1791 __ membar(MacroAssembler::LoadLoad); 1792 __ mov_metadata(t0, exact_klass->constant_encoding()); 1793 __ ld(t1, mdo_addr); 1794 __ xorr(t1, t0, t1); 1795 __ andi(t1, t1, TypeEntries::type_mask); 1796 __ beqz(t1, ok); 1797 1798 __ stop("unexpected profiling mismatch"); 1799 __ bind(ok); 1800 } 1801 #endif 1802 // first time here. Set profile type. 1803 __ sd(tmp, mdo_addr); 1804 #ifdef ASSERT 1805 __ andi(tmp, tmp, TypeEntries::type_mask); 1806 __ verify_klass_ptr(tmp); 1807 #endif 1808 } else { 1809 assert(ciTypeEntries::valid_ciklass(current_klass) != nullptr && 1810 ciTypeEntries::valid_ciklass(current_klass) != exact_klass, "inconsistent"); 1811 1812 __ ld(tmp, mdo_addr); 1813 // already unknown. Nothing to do anymore. 1814 __ test_bit(t0, tmp, exact_log2(TypeEntries::type_unknown)); 1815 __ bnez(t0, next); 1816 1817 __ ori(tmp, tmp, TypeEntries::type_unknown); 1818 __ sd(tmp, mdo_addr); 1819 } 1820 } 1821 1822 void LIR_Assembler::check_null(Register tmp, Label &update, intptr_t current_klass, 1823 Address mdo_addr, bool do_update, Label &next) { 1824 __ bnez(tmp, update); 1825 if (!TypeEntries::was_null_seen(current_klass)) { 1826 __ ld(t1, mdo_addr); 1827 __ ori(t1, t1, TypeEntries::null_seen); 1828 __ sd(t1, mdo_addr); 1829 } 1830 if (do_update) { 1831 __ j(next); 1832 } 1833 } 1834 1835 void LIR_Assembler::emit_profile_type(LIR_OpProfileType* op) { 1836 COMMENT("emit_profile_type {"); 1837 Register obj = op->obj()->as_register(); 1838 Register tmp = op->tmp()->as_pointer_register(); 1839 Address mdo_addr = as_Address(op->mdp()->as_address_ptr()); 1840 ciKlass* exact_klass = op->exact_klass(); 1841 intptr_t current_klass = op->current_klass(); 1842 bool not_null = op->not_null(); 1843 bool no_conflict = op->no_conflict(); 1844 1845 Label update, next, none; 1846 1847 bool do_null = !not_null; 1848 bool exact_klass_set = exact_klass != nullptr && ciTypeEntries::valid_ciklass(current_klass) == exact_klass; 1849 bool do_update = !TypeEntries::is_type_unknown(current_klass) && !exact_klass_set; 1850 1851 assert(do_null || do_update, "why are we here?"); 1852 assert(!TypeEntries::was_null_seen(current_klass) || do_update, "why are we here?"); 1853 assert_different_registers(tmp, t0, t1, mdo_addr.base()); 1854 1855 __ verify_oop(obj); 1856 1857 if (tmp != obj) { 1858 __ mv(tmp, obj); 1859 } 1860 if (do_null) { 1861 check_null(tmp, update, current_klass, mdo_addr, do_update, next); 1862 #ifdef ASSERT 1863 } else { 1864 __ bnez(tmp, update); 1865 __ stop("unexpected null obj"); 1866 #endif 1867 } 1868 1869 __ bind(update); 1870 1871 if (do_update) { 1872 #ifdef ASSERT 1873 if (exact_klass != nullptr) { 1874 check_exact_klass(tmp, exact_klass); 1875 } 1876 #endif 1877 if (!no_conflict) { 1878 check_conflict(exact_klass, current_klass, tmp, next, none, mdo_addr); 1879 } else { 1880 check_no_conflict(exact_klass, current_klass, tmp, mdo_addr, next); 1881 } 1882 1883 __ bind(next); 1884 } 1885 COMMENT("} emit_profile_type"); 1886 } 1887 1888 void LIR_Assembler::align_backward_branch_target() { } 1889 1890 void LIR_Assembler::negate(LIR_Opr left, LIR_Opr dest, LIR_Opr tmp) { 1891 // tmp must be unused 1892 assert(tmp->is_illegal(), "wasting a register if tmp is allocated"); 1893 1894 if (left->is_single_cpu()) { 1895 assert(dest->is_single_cpu(), "expect single result reg"); 1896 __ negw(dest->as_register(), left->as_register()); 1897 } else if (left->is_double_cpu()) { 1898 assert(dest->is_double_cpu(), "expect double result reg"); 1899 __ neg(dest->as_register_lo(), left->as_register_lo()); 1900 } else if (left->is_single_fpu()) { 1901 assert(dest->is_single_fpu(), "expect single float result reg"); 1902 __ fneg_s(dest->as_float_reg(), left->as_float_reg()); 1903 } else { 1904 assert(left->is_double_fpu(), "expect double float operand reg"); 1905 assert(dest->is_double_fpu(), "expect double float result reg"); 1906 __ fneg_d(dest->as_double_reg(), left->as_double_reg()); 1907 } 1908 } 1909 1910 1911 void LIR_Assembler::leal(LIR_Opr addr, LIR_Opr dest, LIR_PatchCode patch_code, CodeEmitInfo* info) { 1912 if (patch_code != lir_patch_none) { 1913 deoptimize_trap(info); 1914 return; 1915 } 1916 1917 LIR_Address* adr = addr->as_address_ptr(); 1918 Register dst = dest->as_pointer_register(); 1919 1920 assert_different_registers(dst, t0); 1921 if (adr->base()->is_valid() && dst == adr->base()->as_pointer_register() && (!adr->index()->is_cpu_register())) { 1922 int scale = adr->scale(); 1923 intptr_t offset = adr->disp(); 1924 LIR_Opr index_op = adr->index(); 1925 if (index_op->is_constant()) { 1926 offset += ((intptr_t)index_op->as_constant_ptr()->as_jint()) << scale; 1927 } 1928 1929 if (!Assembler::is_simm12(offset)) { 1930 __ la(t0, as_Address(adr)); 1931 __ mv(dst, t0); 1932 return; 1933 } 1934 } 1935 1936 __ la(dst, as_Address(adr)); 1937 } 1938 1939 1940 void LIR_Assembler::rt_call(LIR_Opr result, address dest, const LIR_OprList* args, LIR_Opr tmp, CodeEmitInfo* info) { 1941 assert(!tmp->is_valid(), "don't need temporary"); 1942 1943 Assembler::IncompressibleScope scope(_masm); 1944 // Post call nops must be natural aligned due to cmodx rules. 1945 align_call(lir_rtcall); 1946 1947 __ rt_call(dest); 1948 1949 if (info != nullptr) { 1950 add_call_info_here(info); 1951 } 1952 __ post_call_nop(); 1953 } 1954 1955 void LIR_Assembler::volatile_move_op(LIR_Opr src, LIR_Opr dest, BasicType type, CodeEmitInfo* info) { 1956 if (dest->is_address() || src->is_address()) { 1957 move_op(src, dest, type, lir_patch_none, info, /* wide */ false); 1958 } else { 1959 ShouldNotReachHere(); 1960 } 1961 } 1962 1963 #ifdef ASSERT 1964 // emit run-time assertion 1965 void LIR_Assembler::emit_assert(LIR_OpAssert* op) { 1966 assert(op->code() == lir_assert, "must be"); 1967 1968 Label ok; 1969 if (op->in_opr1()->is_valid()) { 1970 assert(op->in_opr2()->is_valid(), "both operands must be valid"); 1971 bool is_unordered = false; 1972 LIR_Condition cond = op->condition(); 1973 emit_branch(cond, op->in_opr1(), op->in_opr2(), ok, /* is_far */ false, 1974 /* is_unordered */(cond == lir_cond_greaterEqual || cond == lir_cond_greater) ? false : true); 1975 } else { 1976 assert(op->in_opr2()->is_illegal(), "both operands must be illegal"); 1977 assert(op->condition() == lir_cond_always, "no other conditions allowed"); 1978 } 1979 1980 if (op->halt()) { 1981 const char* str = __ code_string(op->msg()); 1982 __ stop(str); 1983 } else { 1984 breakpoint(); 1985 } 1986 __ bind(ok); 1987 } 1988 #endif 1989 1990 #ifndef PRODUCT 1991 #define COMMENT(x) do { __ block_comment(x); } while (0) 1992 #else 1993 #define COMMENT(x) 1994 #endif 1995 1996 void LIR_Assembler::membar() { 1997 COMMENT("membar"); 1998 __ membar(MacroAssembler::AnyAny); 1999 } 2000 2001 void LIR_Assembler::membar_acquire() { 2002 __ membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); 2003 } 2004 2005 void LIR_Assembler::membar_release() { 2006 __ membar(MacroAssembler::LoadStore | MacroAssembler::StoreStore); 2007 } 2008 2009 void LIR_Assembler::membar_loadload() { 2010 __ membar(MacroAssembler::LoadLoad); 2011 } 2012 2013 void LIR_Assembler::membar_storestore() { 2014 __ membar(MacroAssembler::StoreStore); 2015 } 2016 2017 void LIR_Assembler::membar_loadstore() { __ membar(MacroAssembler::LoadStore); } 2018 2019 void LIR_Assembler::membar_storeload() { __ membar(MacroAssembler::StoreLoad); } 2020 2021 void LIR_Assembler::on_spin_wait() { 2022 __ pause(); 2023 } 2024 2025 void LIR_Assembler::get_thread(LIR_Opr result_reg) { 2026 __ mv(result_reg->as_register(), xthread); 2027 } 2028 2029 void LIR_Assembler::peephole(LIR_List *lir) {} 2030 2031 void LIR_Assembler::atomic_op(LIR_Code code, LIR_Opr src, LIR_Opr data, LIR_Opr dest, LIR_Opr tmp_op) { 2032 Address addr = as_Address(src->as_address_ptr()); 2033 BasicType type = src->type(); 2034 bool is_oop = is_reference_type(type); 2035 2036 get_op(type); 2037 2038 switch (code) { 2039 case lir_xadd: 2040 { 2041 RegisterOrConstant inc; 2042 Register tmp = as_reg(tmp_op); 2043 Register dst = as_reg(dest); 2044 if (data->is_constant()) { 2045 inc = RegisterOrConstant(as_long(data)); 2046 assert_different_registers(dst, addr.base(), tmp); 2047 assert_different_registers(tmp, t0); 2048 } else { 2049 inc = RegisterOrConstant(as_reg(data)); 2050 assert_different_registers(inc.as_register(), dst, addr.base(), tmp); 2051 } 2052 __ la(tmp, addr); 2053 (_masm->*add)(dst, inc, tmp); 2054 break; 2055 } 2056 case lir_xchg: 2057 { 2058 Register tmp = tmp_op->as_register(); 2059 Register obj = as_reg(data); 2060 Register dst = as_reg(dest); 2061 if (is_oop && UseCompressedOops) { 2062 __ encode_heap_oop(t0, obj); 2063 obj = t0; 2064 } 2065 assert_different_registers(obj, addr.base(), tmp); 2066 assert_different_registers(dst, addr.base(), tmp); 2067 __ la(tmp, addr); 2068 (_masm->*xchg)(dst, obj, tmp); 2069 if (is_oop && UseCompressedOops) { 2070 __ decode_heap_oop(dst); 2071 } 2072 } 2073 break; 2074 default: 2075 ShouldNotReachHere(); 2076 } 2077 __ membar(MacroAssembler::AnyAny); 2078 } 2079 2080 int LIR_Assembler::array_element_size(BasicType type) const { 2081 int elem_size = type2aelembytes(type); 2082 return exact_log2(elem_size); 2083 } 2084 2085 // helper functions which checks for overflow and sets bailout if it 2086 // occurs. Always returns a valid embeddable pointer but in the 2087 // bailout case the pointer won't be to unique storage. 2088 address LIR_Assembler::float_constant(float f) { 2089 address const_addr = __ float_constant(f); 2090 if (const_addr == nullptr) { 2091 bailout("const section overflow"); 2092 return __ code()->consts()->start(); 2093 } else { 2094 return const_addr; 2095 } 2096 } 2097 2098 address LIR_Assembler::double_constant(double d) { 2099 address const_addr = __ double_constant(d); 2100 if (const_addr == nullptr) { 2101 bailout("const section overflow"); 2102 return __ code()->consts()->start(); 2103 } else { 2104 return const_addr; 2105 } 2106 } 2107 2108 address LIR_Assembler::int_constant(jlong n) { 2109 address const_addr = __ long_constant(n); 2110 if (const_addr == nullptr) { 2111 bailout("const section overflow"); 2112 return __ code()->consts()->start(); 2113 } else { 2114 return const_addr; 2115 } 2116 } 2117 2118 void LIR_Assembler::casw(Register addr, Register newval, Register cmpval) { 2119 __ cmpxchg(addr, cmpval, newval, Assembler::int32, Assembler::aq /* acquire */, 2120 Assembler::rl /* release */, t0, true /* result as bool */); 2121 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1 2122 __ membar(MacroAssembler::AnyAny); 2123 } 2124 2125 void LIR_Assembler::caswu(Register addr, Register newval, Register cmpval) { 2126 __ cmpxchg(addr, cmpval, newval, Assembler::uint32, Assembler::aq /* acquire */, 2127 Assembler::rl /* release */, t0, true /* result as bool */); 2128 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1 2129 __ membar(MacroAssembler::AnyAny); 2130 } 2131 2132 void LIR_Assembler::casl(Register addr, Register newval, Register cmpval) { 2133 __ cmpxchg(addr, cmpval, newval, Assembler::int64, Assembler::aq /* acquire */, 2134 Assembler::rl /* release */, t0, true /* result as bool */); 2135 __ seqz(t0, t0); // cmpxchg not equal, set t0 to 1 2136 __ membar(MacroAssembler::AnyAny); 2137 } 2138 2139 void LIR_Assembler::deoptimize_trap(CodeEmitInfo *info) { 2140 address target = nullptr; 2141 2142 switch (patching_id(info)) { 2143 case PatchingStub::access_field_id: 2144 target = Runtime1::entry_for(StubId::c1_access_field_patching_id); 2145 break; 2146 case PatchingStub::load_klass_id: 2147 target = Runtime1::entry_for(StubId::c1_load_klass_patching_id); 2148 break; 2149 case PatchingStub::load_mirror_id: 2150 target = Runtime1::entry_for(StubId::c1_load_mirror_patching_id); 2151 break; 2152 case PatchingStub::load_appendix_id: 2153 target = Runtime1::entry_for(StubId::c1_load_appendix_patching_id); 2154 break; 2155 default: ShouldNotReachHere(); 2156 } 2157 2158 __ far_call(RuntimeAddress(target)); 2159 add_call_info_here(info); 2160 } 2161 2162 void LIR_Assembler::check_exact_klass(Register tmp, ciKlass* exact_klass) { 2163 Label ok; 2164 __ load_klass(tmp, tmp); 2165 __ mov_metadata(t0, exact_klass->constant_encoding()); 2166 __ beq(tmp, t0, ok); 2167 __ stop("exact klass and actual klass differ"); 2168 __ bind(ok); 2169 } 2170 2171 void LIR_Assembler::get_op(BasicType type) { 2172 switch (type) { 2173 case T_INT: 2174 xchg = &MacroAssembler::atomic_xchgalw; 2175 add = &MacroAssembler::atomic_addalw; 2176 break; 2177 case T_LONG: 2178 xchg = &MacroAssembler::atomic_xchgal; 2179 add = &MacroAssembler::atomic_addal; 2180 break; 2181 case T_OBJECT: 2182 case T_ARRAY: 2183 if (UseCompressedOops) { 2184 xchg = &MacroAssembler::atomic_xchgalwu; 2185 add = &MacroAssembler::atomic_addalw; 2186 } else { 2187 xchg = &MacroAssembler::atomic_xchgal; 2188 add = &MacroAssembler::atomic_addal; 2189 } 2190 break; 2191 default: 2192 ShouldNotReachHere(); 2193 } 2194 } 2195 2196 // emit_opTypeCheck sub functions 2197 void LIR_Assembler::typecheck_lir_store(LIR_OpTypeCheck* op, bool should_profile) { 2198 Register value = op->object()->as_register(); 2199 Register array = op->array()->as_register(); 2200 Register k_RInfo = op->tmp1()->as_register(); 2201 Register klass_RInfo = op->tmp2()->as_register(); 2202 Register Rtmp1 = op->tmp3()->as_register(); 2203 2204 CodeStub* stub = op->stub(); 2205 2206 // check if it needs to be profiled 2207 ciMethodData* md = nullptr; 2208 ciProfileData* data = nullptr; 2209 2210 if (should_profile) { 2211 data_check(op, &md, &data); 2212 } 2213 Label done; 2214 Label* success_target = &done; 2215 Label* failure_target = stub->entry(); 2216 2217 if (should_profile) { 2218 profile_object(md, data, value, k_RInfo, klass_RInfo, &done); 2219 } else { 2220 __ beqz(value, done); 2221 } 2222 2223 add_debug_info_for_null_check_here(op->info_for_exception()); 2224 __ load_klass(k_RInfo, array); 2225 __ load_klass(klass_RInfo, value); 2226 2227 lir_store_slowcheck(k_RInfo, klass_RInfo, Rtmp1, success_target, failure_target); 2228 2229 __ bind(done); 2230 } 2231 2232 void LIR_Assembler::lir_store_slowcheck(Register k_RInfo, Register klass_RInfo, Register Rtmp1, 2233 Label* success_target, Label* failure_target) { 2234 // get instance klass (it's already uncompressed) 2235 __ ld(k_RInfo, Address(k_RInfo, ObjArrayKlass::element_klass_offset())); 2236 // perform the fast part of the checking logic 2237 __ check_klass_subtype_fast_path(klass_RInfo, k_RInfo, Rtmp1, success_target, failure_target, nullptr); 2238 // call out-of-line instance of __ check_klass_subtype_slow_path(...) 2239 __ subi(sp, sp, 2 * wordSize); // 2: store k_RInfo and klass_RInfo 2240 __ sd(klass_RInfo, Address(sp, wordSize)); // sub klass 2241 __ sd(k_RInfo, Address(sp, 0)); // super klass 2242 __ far_call(RuntimeAddress(Runtime1::entry_for(StubId::c1_slow_subtype_check_id))); 2243 // load result to k_RInfo 2244 __ ld(k_RInfo, Address(sp, 0)); 2245 __ addi(sp, sp, 2 * wordSize); // 2: pop out k_RInfo and klass_RInfo 2246 // result is a boolean 2247 __ beqz(k_RInfo, *failure_target, /* is_far */ true); 2248 } 2249 2250 void LIR_Assembler::const2reg_helper(LIR_Opr src) { 2251 switch (src->as_constant_ptr()->type()) { 2252 case T_INT: 2253 case T_ADDRESS: 2254 case T_OBJECT: 2255 case T_ARRAY: 2256 case T_METADATA: 2257 const2reg(src, FrameMap::t0_opr, lir_patch_none, nullptr); 2258 break; 2259 case T_LONG: 2260 const2reg(src, FrameMap::t0_long_opr, lir_patch_none, nullptr); 2261 break; 2262 case T_FLOAT: 2263 case T_DOUBLE: 2264 default: 2265 ShouldNotReachHere(); 2266 } 2267 } 2268 2269 void LIR_Assembler::logic_op_reg32(Register dst, Register left, Register right, LIR_Code code) { 2270 switch (code) { 2271 case lir_logic_and: __ andrw(dst, left, right); break; 2272 case lir_logic_or: __ orrw (dst, left, right); break; 2273 case lir_logic_xor: __ xorrw(dst, left, right); break; 2274 default: ShouldNotReachHere(); 2275 } 2276 } 2277 2278 void LIR_Assembler::logic_op_reg(Register dst, Register left, Register right, LIR_Code code) { 2279 switch (code) { 2280 case lir_logic_and: __ andr(dst, left, right); break; 2281 case lir_logic_or: __ orr (dst, left, right); break; 2282 case lir_logic_xor: __ xorr(dst, left, right); break; 2283 default: ShouldNotReachHere(); 2284 } 2285 } 2286 2287 void LIR_Assembler::logic_op_imm(Register dst, Register left, int right, LIR_Code code) { 2288 switch (code) { 2289 case lir_logic_and: __ andi(dst, left, right); break; 2290 case lir_logic_or: __ ori (dst, left, right); break; 2291 case lir_logic_xor: __ xori(dst, left, right); break; 2292 default: ShouldNotReachHere(); 2293 } 2294 } 2295 2296 void LIR_Assembler::store_parameter(Register r, int offset_from_rsp_in_words) { 2297 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2298 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2299 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2300 __ sd(r, Address(sp, offset_from_rsp_in_bytes)); 2301 } 2302 2303 void LIR_Assembler::store_parameter(jint c, int offset_from_rsp_in_words) { 2304 assert(offset_from_rsp_in_words >= 0, "invalid offset from rsp"); 2305 int offset_from_rsp_in_bytes = offset_from_rsp_in_words * BytesPerWord; 2306 assert(offset_from_rsp_in_bytes < frame_map()->reserved_argument_area_size(), "invalid offset"); 2307 __ mv(t0, c); 2308 __ sd(t0, Address(sp, offset_from_rsp_in_bytes)); 2309 } 2310 2311 #undef __ --- EOF ---