1 /* 2 * Copyright (c) 1997, 2024, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. 4 * Copyright (c) 2020, 2024, Huawei Technologies Co., Ltd. All rights reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 #include "asm/assembler.hpp" 29 #include "asm/assembler.inline.hpp" 30 #include "code/compiledIC.hpp" 31 #include "compiler/disassembler.hpp" 32 #include "gc/shared/barrierSet.hpp" 33 #include "gc/shared/barrierSetAssembler.hpp" 34 #include "gc/shared/cardTable.hpp" 35 #include "gc/shared/cardTableBarrierSet.hpp" 36 #include "gc/shared/collectedHeap.hpp" 37 #include "interpreter/bytecodeHistogram.hpp" 38 #include "interpreter/interpreter.hpp" 39 #include "memory/resourceArea.hpp" 40 #include "memory/universe.hpp" 41 #include "oops/accessDecorators.hpp" 42 #include "oops/compressedKlass.inline.hpp" 43 #include "oops/compressedOops.inline.hpp" 44 #include "oops/klass.inline.hpp" 45 #include "oops/oop.hpp" 46 #include "runtime/interfaceSupport.inline.hpp" 47 #include "runtime/javaThread.hpp" 48 #include "runtime/jniHandles.inline.hpp" 49 #include "runtime/sharedRuntime.hpp" 50 #include "runtime/stubRoutines.hpp" 51 #include "utilities/globalDefinitions.hpp" 52 #include "utilities/powerOfTwo.hpp" 53 #ifdef COMPILER2 54 #include "opto/compile.hpp" 55 #include "opto/node.hpp" 56 #include "opto/output.hpp" 57 #endif 58 59 #ifdef PRODUCT 60 #define BLOCK_COMMENT(str) /* nothing */ 61 #else 62 #define BLOCK_COMMENT(str) block_comment(str) 63 #endif 64 #define STOP(str) stop(str); 65 #define BIND(label) bind(label); __ BLOCK_COMMENT(#label ":") 66 67 68 69 Register MacroAssembler::extract_rs1(address instr) { 70 assert_cond(instr != nullptr); 71 return as_Register(Assembler::extract(Assembler::ld_instr(instr), 19, 15)); 72 } 73 74 Register MacroAssembler::extract_rs2(address instr) { 75 assert_cond(instr != nullptr); 76 return as_Register(Assembler::extract(Assembler::ld_instr(instr), 24, 20)); 77 } 78 79 Register MacroAssembler::extract_rd(address instr) { 80 assert_cond(instr != nullptr); 81 return as_Register(Assembler::extract(Assembler::ld_instr(instr), 11, 7)); 82 } 83 84 uint32_t MacroAssembler::extract_opcode(address instr) { 85 assert_cond(instr != nullptr); 86 return Assembler::extract(Assembler::ld_instr(instr), 6, 0); 87 } 88 89 uint32_t MacroAssembler::extract_funct3(address instr) { 90 assert_cond(instr != nullptr); 91 return Assembler::extract(Assembler::ld_instr(instr), 14, 12); 92 } 93 94 bool MacroAssembler::is_pc_relative_at(address instr) { 95 // auipc + jalr 96 // auipc + addi 97 // auipc + load 98 // auipc + fload_load 99 return (is_auipc_at(instr)) && 100 (is_addi_at(instr + instruction_size) || 101 is_jalr_at(instr + instruction_size) || 102 is_load_at(instr + instruction_size) || 103 is_float_load_at(instr + instruction_size)) && 104 check_pc_relative_data_dependency(instr); 105 } 106 107 // ie:ld(Rd, Label) 108 bool MacroAssembler::is_load_pc_relative_at(address instr) { 109 return is_auipc_at(instr) && // auipc 110 is_ld_at(instr + instruction_size) && // ld 111 check_load_pc_relative_data_dependency(instr); 112 } 113 114 bool MacroAssembler::is_movptr1_at(address instr) { 115 return is_lui_at(instr) && // Lui 116 is_addi_at(instr + instruction_size) && // Addi 117 is_slli_shift_at(instr + instruction_size * 2, 11) && // Slli Rd, Rs, 11 118 is_addi_at(instr + instruction_size * 3) && // Addi 119 is_slli_shift_at(instr + instruction_size * 4, 6) && // Slli Rd, Rs, 6 120 (is_addi_at(instr + instruction_size * 5) || 121 is_jalr_at(instr + instruction_size * 5) || 122 is_load_at(instr + instruction_size * 5)) && // Addi/Jalr/Load 123 check_movptr1_data_dependency(instr); 124 } 125 126 bool MacroAssembler::is_movptr2_at(address instr) { 127 return is_lui_at(instr) && // lui 128 is_lui_at(instr + instruction_size) && // lui 129 is_slli_shift_at(instr + instruction_size * 2, 18) && // slli Rd, Rs, 18 130 is_add_at(instr + instruction_size * 3) && 131 (is_addi_at(instr + instruction_size * 4) || 132 is_jalr_at(instr + instruction_size * 4) || 133 is_load_at(instr + instruction_size * 4)) && // Addi/Jalr/Load 134 check_movptr2_data_dependency(instr); 135 } 136 137 bool MacroAssembler::is_li16u_at(address instr) { 138 return is_lui_at(instr) && // lui 139 is_srli_at(instr + instruction_size) && // srli 140 check_li16u_data_dependency(instr); 141 } 142 143 bool MacroAssembler::is_li32_at(address instr) { 144 return is_lui_at(instr) && // lui 145 is_addiw_at(instr + instruction_size) && // addiw 146 check_li32_data_dependency(instr); 147 } 148 149 bool MacroAssembler::is_lwu_to_zr(address instr) { 150 assert_cond(instr != nullptr); 151 return (extract_opcode(instr) == 0b0000011 && 152 extract_funct3(instr) == 0b110 && 153 extract_rd(instr) == zr); // zr 154 } 155 156 uint32_t MacroAssembler::get_membar_kind(address addr) { 157 assert_cond(addr != nullptr); 158 assert(is_membar(addr), "no membar found"); 159 160 uint32_t insn = Bytes::get_native_u4(addr); 161 162 uint32_t predecessor = Assembler::extract(insn, 27, 24); 163 uint32_t successor = Assembler::extract(insn, 23, 20); 164 165 return MacroAssembler::pred_succ_to_membar_mask(predecessor, successor); 166 } 167 168 void MacroAssembler::set_membar_kind(address addr, uint32_t order_kind) { 169 assert_cond(addr != nullptr); 170 assert(is_membar(addr), "no membar found"); 171 172 uint32_t predecessor = 0; 173 uint32_t successor = 0; 174 175 MacroAssembler::membar_mask_to_pred_succ(order_kind, predecessor, successor); 176 177 uint32_t insn = Bytes::get_native_u4(addr); 178 address pInsn = (address) &insn; 179 Assembler::patch(pInsn, 27, 24, predecessor); 180 Assembler::patch(pInsn, 23, 20, successor); 181 182 address membar = addr; 183 Assembler::sd_instr(membar, insn); 184 } 185 186 187 static void pass_arg0(MacroAssembler* masm, Register arg) { 188 if (c_rarg0 != arg) { 189 masm->mv(c_rarg0, arg); 190 } 191 } 192 193 static void pass_arg1(MacroAssembler* masm, Register arg) { 194 if (c_rarg1 != arg) { 195 masm->mv(c_rarg1, arg); 196 } 197 } 198 199 static void pass_arg2(MacroAssembler* masm, Register arg) { 200 if (c_rarg2 != arg) { 201 masm->mv(c_rarg2, arg); 202 } 203 } 204 205 static void pass_arg3(MacroAssembler* masm, Register arg) { 206 if (c_rarg3 != arg) { 207 masm->mv(c_rarg3, arg); 208 } 209 } 210 211 void MacroAssembler::push_cont_fastpath(Register java_thread) { 212 if (!Continuations::enabled()) return; 213 Label done; 214 ld(t0, Address(java_thread, JavaThread::cont_fastpath_offset())); 215 bleu(sp, t0, done); 216 sd(sp, Address(java_thread, JavaThread::cont_fastpath_offset())); 217 bind(done); 218 } 219 220 void MacroAssembler::pop_cont_fastpath(Register java_thread) { 221 if (!Continuations::enabled()) return; 222 Label done; 223 ld(t0, Address(java_thread, JavaThread::cont_fastpath_offset())); 224 bltu(sp, t0, done); 225 sd(zr, Address(java_thread, JavaThread::cont_fastpath_offset())); 226 bind(done); 227 } 228 229 int MacroAssembler::align(int modulus, int extra_offset) { 230 CompressibleRegion cr(this); 231 intptr_t before = offset(); 232 while ((offset() + extra_offset) % modulus != 0) { nop(); } 233 return (int)(offset() - before); 234 } 235 236 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 237 call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions); 238 } 239 240 // Implementation of call_VM versions 241 242 void MacroAssembler::call_VM(Register oop_result, 243 address entry_point, 244 bool check_exceptions) { 245 call_VM_helper(oop_result, entry_point, 0, check_exceptions); 246 } 247 248 void MacroAssembler::call_VM(Register oop_result, 249 address entry_point, 250 Register arg_1, 251 bool check_exceptions) { 252 pass_arg1(this, arg_1); 253 call_VM_helper(oop_result, entry_point, 1, check_exceptions); 254 } 255 256 void MacroAssembler::call_VM(Register oop_result, 257 address entry_point, 258 Register arg_1, 259 Register arg_2, 260 bool check_exceptions) { 261 assert_different_registers(arg_1, c_rarg2); 262 pass_arg2(this, arg_2); 263 pass_arg1(this, arg_1); 264 call_VM_helper(oop_result, entry_point, 2, check_exceptions); 265 } 266 267 void MacroAssembler::call_VM(Register oop_result, 268 address entry_point, 269 Register arg_1, 270 Register arg_2, 271 Register arg_3, 272 bool check_exceptions) { 273 assert_different_registers(arg_1, c_rarg2, c_rarg3); 274 assert_different_registers(arg_2, c_rarg3); 275 pass_arg3(this, arg_3); 276 277 pass_arg2(this, arg_2); 278 279 pass_arg1(this, arg_1); 280 call_VM_helper(oop_result, entry_point, 3, check_exceptions); 281 } 282 283 void MacroAssembler::call_VM(Register oop_result, 284 Register last_java_sp, 285 address entry_point, 286 int number_of_arguments, 287 bool check_exceptions) { 288 call_VM_base(oop_result, xthread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 289 } 290 291 void MacroAssembler::call_VM(Register oop_result, 292 Register last_java_sp, 293 address entry_point, 294 Register arg_1, 295 bool check_exceptions) { 296 pass_arg1(this, arg_1); 297 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 298 } 299 300 void MacroAssembler::call_VM(Register oop_result, 301 Register last_java_sp, 302 address entry_point, 303 Register arg_1, 304 Register arg_2, 305 bool check_exceptions) { 306 307 assert_different_registers(arg_1, c_rarg2); 308 pass_arg2(this, arg_2); 309 pass_arg1(this, arg_1); 310 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 311 } 312 313 void MacroAssembler::call_VM(Register oop_result, 314 Register last_java_sp, 315 address entry_point, 316 Register arg_1, 317 Register arg_2, 318 Register arg_3, 319 bool check_exceptions) { 320 assert_different_registers(arg_1, c_rarg2, c_rarg3); 321 assert_different_registers(arg_2, c_rarg3); 322 pass_arg3(this, arg_3); 323 pass_arg2(this, arg_2); 324 pass_arg1(this, arg_1); 325 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 326 } 327 328 void MacroAssembler::post_call_nop() { 329 if (!Continuations::enabled()) { 330 return; 331 } 332 relocate(post_call_nop_Relocation::spec(), [&] { 333 InlineSkippedInstructionsCounter skipCounter(this); 334 nop(); 335 li32(zr, 0); 336 }); 337 } 338 339 // these are no-ops overridden by InterpreterMacroAssembler 340 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {} 341 void MacroAssembler::check_and_handle_popframe(Register java_thread) {} 342 343 // Calls to C land 344 // 345 // When entering C land, the fp, & esp of the last Java frame have to be recorded 346 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp 347 // has to be reset to 0. This is required to allow proper stack traversal. 348 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 349 Register last_java_fp, 350 Register last_java_pc) { 351 352 if (last_java_pc->is_valid()) { 353 sd(last_java_pc, Address(xthread, 354 JavaThread::frame_anchor_offset() + 355 JavaFrameAnchor::last_Java_pc_offset())); 356 } 357 358 // determine last_java_sp register 359 if (!last_java_sp->is_valid()) { 360 last_java_sp = esp; 361 } 362 363 sd(last_java_sp, Address(xthread, JavaThread::last_Java_sp_offset())); 364 365 // last_java_fp is optional 366 if (last_java_fp->is_valid()) { 367 sd(last_java_fp, Address(xthread, JavaThread::last_Java_fp_offset())); 368 } 369 } 370 371 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 372 Register last_java_fp, 373 address last_java_pc, 374 Register tmp) { 375 assert(last_java_pc != nullptr, "must provide a valid PC"); 376 377 la(tmp, last_java_pc); 378 sd(tmp, Address(xthread, JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset())); 379 380 set_last_Java_frame(last_java_sp, last_java_fp, noreg); 381 } 382 383 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 384 Register last_java_fp, 385 Label &L, 386 Register tmp) { 387 if (L.is_bound()) { 388 set_last_Java_frame(last_java_sp, last_java_fp, target(L), tmp); 389 } else { 390 L.add_patch_at(code(), locator()); 391 IncompressibleRegion ir(this); // the label address will be patched back. 392 set_last_Java_frame(last_java_sp, last_java_fp, pc() /* Patched later */, tmp); 393 } 394 } 395 396 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { 397 // we must set sp to zero to clear frame 398 sd(zr, Address(xthread, JavaThread::last_Java_sp_offset())); 399 400 // must clear fp, so that compiled frames are not confused; it is 401 // possible that we need it only for debugging 402 if (clear_fp) { 403 sd(zr, Address(xthread, JavaThread::last_Java_fp_offset())); 404 } 405 406 // Always clear the pc because it could have been set by make_walkable() 407 sd(zr, Address(xthread, JavaThread::last_Java_pc_offset())); 408 } 409 410 void MacroAssembler::call_VM_base(Register oop_result, 411 Register java_thread, 412 Register last_java_sp, 413 address entry_point, 414 int number_of_arguments, 415 bool check_exceptions) { 416 // determine java_thread register 417 if (!java_thread->is_valid()) { 418 java_thread = xthread; 419 } 420 // determine last_java_sp register 421 if (!last_java_sp->is_valid()) { 422 last_java_sp = esp; 423 } 424 425 // debugging support 426 assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); 427 assert(java_thread == xthread, "unexpected register"); 428 429 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); 430 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); 431 432 // push java thread (becomes first argument of C function) 433 mv(c_rarg0, java_thread); 434 435 // set last Java frame before call 436 assert(last_java_sp != fp, "can't use fp"); 437 438 Label l; 439 set_last_Java_frame(last_java_sp, fp, l, t0); 440 441 // do the call, remove parameters 442 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l); 443 444 // reset last Java frame 445 // Only interpreter should have to clear fp 446 reset_last_Java_frame(true); 447 448 // C++ interp handles this in the interpreter 449 check_and_handle_popframe(java_thread); 450 check_and_handle_earlyret(java_thread); 451 452 if (check_exceptions) { 453 // check for pending exceptions (java_thread is set upon return) 454 ld(t0, Address(java_thread, in_bytes(Thread::pending_exception_offset()))); 455 Label ok; 456 beqz(t0, ok); 457 RuntimeAddress target(StubRoutines::forward_exception_entry()); 458 relocate(target.rspec(), [&] { 459 int32_t offset; 460 la(t0, target.target(), offset); 461 jr(t0, offset); 462 }); 463 bind(ok); 464 } 465 466 // get oop result if there is one and reset the value in the thread 467 if (oop_result->is_valid()) { 468 get_vm_result(oop_result, java_thread); 469 } 470 } 471 472 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { 473 ld(oop_result, Address(java_thread, JavaThread::vm_result_offset())); 474 sd(zr, Address(java_thread, JavaThread::vm_result_offset())); 475 verify_oop_msg(oop_result, "broken oop in call_VM_base"); 476 } 477 478 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { 479 ld(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); 480 sd(zr, Address(java_thread, JavaThread::vm_result_2_offset())); 481 } 482 483 void MacroAssembler::clinit_barrier(Register klass, Register tmp, Label* L_fast_path, Label* L_slow_path) { 484 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required"); 485 assert_different_registers(klass, xthread, tmp); 486 487 Label L_fallthrough, L_tmp; 488 if (L_fast_path == nullptr) { 489 L_fast_path = &L_fallthrough; 490 } else if (L_slow_path == nullptr) { 491 L_slow_path = &L_fallthrough; 492 } 493 494 // Fast path check: class is fully initialized 495 lbu(tmp, Address(klass, InstanceKlass::init_state_offset())); 496 sub(tmp, tmp, InstanceKlass::fully_initialized); 497 beqz(tmp, *L_fast_path); 498 499 // Fast path check: current thread is initializer thread 500 ld(tmp, Address(klass, InstanceKlass::init_thread_offset())); 501 502 if (L_slow_path == &L_fallthrough) { 503 beq(xthread, tmp, *L_fast_path); 504 bind(*L_slow_path); 505 } else if (L_fast_path == &L_fallthrough) { 506 bne(xthread, tmp, *L_slow_path); 507 bind(*L_fast_path); 508 } else { 509 Unimplemented(); 510 } 511 } 512 513 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) { 514 if (!VerifyOops) { return; } 515 516 // Pass register number to verify_oop_subroutine 517 const char* b = nullptr; 518 { 519 ResourceMark rm; 520 stringStream ss; 521 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line); 522 b = code_string(ss.as_string()); 523 } 524 BLOCK_COMMENT("verify_oop {"); 525 526 push_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); 527 528 mv(c_rarg0, reg); // c_rarg0 : x10 529 { 530 // The length of the instruction sequence emitted should not depend 531 // on the address of the char buffer so that the size of mach nodes for 532 // scratch emit and normal emit matches. 533 IncompressibleRegion ir(this); // Fixed length 534 movptr(t0, (address) b); 535 } 536 537 // call indirectly to solve generation ordering problem 538 RuntimeAddress target(StubRoutines::verify_oop_subroutine_entry_address()); 539 relocate(target.rspec(), [&] { 540 int32_t offset; 541 la(t1, target.target(), offset); 542 ld(t1, Address(t1, offset)); 543 }); 544 jalr(t1); 545 546 pop_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); 547 548 BLOCK_COMMENT("} verify_oop"); 549 } 550 551 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { 552 if (!VerifyOops) { 553 return; 554 } 555 556 const char* b = nullptr; 557 { 558 ResourceMark rm; 559 stringStream ss; 560 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line); 561 b = code_string(ss.as_string()); 562 } 563 BLOCK_COMMENT("verify_oop_addr {"); 564 565 push_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); 566 567 if (addr.uses(sp)) { 568 la(x10, addr); 569 ld(x10, Address(x10, 4 * wordSize)); 570 } else { 571 ld(x10, addr); 572 } 573 574 { 575 // The length of the instruction sequence emitted should not depend 576 // on the address of the char buffer so that the size of mach nodes for 577 // scratch emit and normal emit matches. 578 IncompressibleRegion ir(this); // Fixed length 579 movptr(t0, (address) b); 580 } 581 582 // call indirectly to solve generation ordering problem 583 RuntimeAddress target(StubRoutines::verify_oop_subroutine_entry_address()); 584 relocate(target.rspec(), [&] { 585 int32_t offset; 586 la(t1, target.target(), offset); 587 ld(t1, Address(t1, offset)); 588 }); 589 jalr(t1); 590 591 pop_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); 592 593 BLOCK_COMMENT("} verify_oop_addr"); 594 } 595 596 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, 597 int extra_slot_offset) { 598 // cf. TemplateTable::prepare_invoke(), if (load_receiver). 599 int stackElementSize = Interpreter::stackElementSize; 600 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); 601 #ifdef ASSERT 602 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); 603 assert(offset1 - offset == stackElementSize, "correct arithmetic"); 604 #endif 605 if (arg_slot.is_constant()) { 606 return Address(esp, arg_slot.as_constant() * stackElementSize + offset); 607 } else { 608 assert_different_registers(t0, arg_slot.as_register()); 609 shadd(t0, arg_slot.as_register(), esp, t0, exact_log2(stackElementSize)); 610 return Address(t0, offset); 611 } 612 } 613 614 #ifndef PRODUCT 615 extern "C" void findpc(intptr_t x); 616 #endif 617 618 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) 619 { 620 // In order to get locks to work, we need to fake a in_VM state 621 if (ShowMessageBoxOnError) { 622 JavaThread* thread = JavaThread::current(); 623 JavaThreadState saved_state = thread->thread_state(); 624 thread->set_thread_state(_thread_in_vm); 625 #ifndef PRODUCT 626 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 627 ttyLocker ttyl; 628 BytecodeCounter::print(); 629 } 630 #endif 631 if (os::message_box(msg, "Execution stopped, print registers?")) { 632 ttyLocker ttyl; 633 tty->print_cr(" pc = 0x%016lx", pc); 634 #ifndef PRODUCT 635 tty->cr(); 636 findpc(pc); 637 tty->cr(); 638 #endif 639 tty->print_cr(" x0 = 0x%016lx", regs[0]); 640 tty->print_cr(" x1 = 0x%016lx", regs[1]); 641 tty->print_cr(" x2 = 0x%016lx", regs[2]); 642 tty->print_cr(" x3 = 0x%016lx", regs[3]); 643 tty->print_cr(" x4 = 0x%016lx", regs[4]); 644 tty->print_cr(" x5 = 0x%016lx", regs[5]); 645 tty->print_cr(" x6 = 0x%016lx", regs[6]); 646 tty->print_cr(" x7 = 0x%016lx", regs[7]); 647 tty->print_cr(" x8 = 0x%016lx", regs[8]); 648 tty->print_cr(" x9 = 0x%016lx", regs[9]); 649 tty->print_cr("x10 = 0x%016lx", regs[10]); 650 tty->print_cr("x11 = 0x%016lx", regs[11]); 651 tty->print_cr("x12 = 0x%016lx", regs[12]); 652 tty->print_cr("x13 = 0x%016lx", regs[13]); 653 tty->print_cr("x14 = 0x%016lx", regs[14]); 654 tty->print_cr("x15 = 0x%016lx", regs[15]); 655 tty->print_cr("x16 = 0x%016lx", regs[16]); 656 tty->print_cr("x17 = 0x%016lx", regs[17]); 657 tty->print_cr("x18 = 0x%016lx", regs[18]); 658 tty->print_cr("x19 = 0x%016lx", regs[19]); 659 tty->print_cr("x20 = 0x%016lx", regs[20]); 660 tty->print_cr("x21 = 0x%016lx", regs[21]); 661 tty->print_cr("x22 = 0x%016lx", regs[22]); 662 tty->print_cr("x23 = 0x%016lx", regs[23]); 663 tty->print_cr("x24 = 0x%016lx", regs[24]); 664 tty->print_cr("x25 = 0x%016lx", regs[25]); 665 tty->print_cr("x26 = 0x%016lx", regs[26]); 666 tty->print_cr("x27 = 0x%016lx", regs[27]); 667 tty->print_cr("x28 = 0x%016lx", regs[28]); 668 tty->print_cr("x30 = 0x%016lx", regs[30]); 669 tty->print_cr("x31 = 0x%016lx", regs[31]); 670 BREAKPOINT; 671 } 672 } 673 fatal("DEBUG MESSAGE: %s", msg); 674 } 675 676 void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp2) { 677 assert_different_registers(value, tmp1, tmp2); 678 Label done, tagged, weak_tagged; 679 680 beqz(value, done); // Use null as-is. 681 // Test for tag. 682 andi(tmp1, value, JNIHandles::tag_mask); 683 bnez(tmp1, tagged); 684 685 // Resolve local handle 686 access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp1, tmp2); 687 verify_oop(value); 688 j(done); 689 690 bind(tagged); 691 // Test for jweak tag. 692 STATIC_ASSERT(JNIHandles::TypeTag::weak_global == 0b1); 693 test_bit(tmp1, value, exact_log2(JNIHandles::TypeTag::weak_global)); 694 bnez(tmp1, weak_tagged); 695 696 // Resolve global handle 697 access_load_at(T_OBJECT, IN_NATIVE, value, 698 Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2); 699 verify_oop(value); 700 j(done); 701 702 bind(weak_tagged); 703 // Resolve jweak. 704 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, value, 705 Address(value, -JNIHandles::TypeTag::weak_global), tmp1, tmp2); 706 verify_oop(value); 707 708 bind(done); 709 } 710 711 void MacroAssembler::resolve_global_jobject(Register value, Register tmp1, Register tmp2) { 712 assert_different_registers(value, tmp1, tmp2); 713 Label done; 714 715 beqz(value, done); // Use null as-is. 716 717 #ifdef ASSERT 718 { 719 STATIC_ASSERT(JNIHandles::TypeTag::global == 0b10); 720 Label valid_global_tag; 721 test_bit(tmp1, value, exact_log2(JNIHandles::TypeTag::global)); // Test for global tag. 722 bnez(tmp1, valid_global_tag); 723 stop("non global jobject using resolve_global_jobject"); 724 bind(valid_global_tag); 725 } 726 #endif 727 728 // Resolve global handle 729 access_load_at(T_OBJECT, IN_NATIVE, value, 730 Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2); 731 verify_oop(value); 732 733 bind(done); 734 } 735 736 void MacroAssembler::stop(const char* msg) { 737 BLOCK_COMMENT(msg); 738 illegal_instruction(Assembler::csr::time); 739 emit_int64((uintptr_t)msg); 740 } 741 742 void MacroAssembler::unimplemented(const char* what) { 743 const char* buf = nullptr; 744 { 745 ResourceMark rm; 746 stringStream ss; 747 ss.print("unimplemented: %s", what); 748 buf = code_string(ss.as_string()); 749 } 750 stop(buf); 751 } 752 753 void MacroAssembler::emit_static_call_stub() { 754 IncompressibleRegion ir(this); // Fixed length: see CompiledDirectCall::to_interp_stub_size(). 755 // CompiledDirectCall::set_to_interpreted knows the 756 // exact layout of this stub. 757 758 mov_metadata(xmethod, (Metadata*)nullptr); 759 760 // Jump to the entry point of the c2i stub. 761 int32_t offset = 0; 762 movptr(t0, 0, offset, t1); // lui + lui + slli + add 763 jr(t0, offset); 764 } 765 766 void MacroAssembler::call_VM_leaf_base(address entry_point, 767 int number_of_arguments, 768 Label *retaddr) { 769 int32_t offset = 0; 770 push_reg(RegSet::of(t0, xmethod), sp); // push << t0 & xmethod >> to sp 771 mv(t0, entry_point, offset); 772 jalr(t0, offset); 773 if (retaddr != nullptr) { 774 bind(*retaddr); 775 } 776 pop_reg(RegSet::of(t0, xmethod), sp); // pop << t0 & xmethod >> from sp 777 } 778 779 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { 780 call_VM_leaf_base(entry_point, number_of_arguments); 781 } 782 783 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { 784 pass_arg0(this, arg_0); 785 call_VM_leaf_base(entry_point, 1); 786 } 787 788 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 789 assert_different_registers(arg_1, c_rarg0); 790 pass_arg0(this, arg_0); 791 pass_arg1(this, arg_1); 792 call_VM_leaf_base(entry_point, 2); 793 } 794 795 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, 796 Register arg_1, Register arg_2) { 797 assert_different_registers(arg_1, c_rarg0); 798 assert_different_registers(arg_2, c_rarg0, c_rarg1); 799 pass_arg0(this, arg_0); 800 pass_arg1(this, arg_1); 801 pass_arg2(this, arg_2); 802 call_VM_leaf_base(entry_point, 3); 803 } 804 805 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { 806 pass_arg0(this, arg_0); 807 MacroAssembler::call_VM_leaf_base(entry_point, 1); 808 } 809 810 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 811 812 assert_different_registers(arg_0, c_rarg1); 813 pass_arg1(this, arg_1); 814 pass_arg0(this, arg_0); 815 MacroAssembler::call_VM_leaf_base(entry_point, 2); 816 } 817 818 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 819 assert_different_registers(arg_0, c_rarg1, c_rarg2); 820 assert_different_registers(arg_1, c_rarg2); 821 pass_arg2(this, arg_2); 822 pass_arg1(this, arg_1); 823 pass_arg0(this, arg_0); 824 MacroAssembler::call_VM_leaf_base(entry_point, 3); 825 } 826 827 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 828 assert_different_registers(arg_0, c_rarg1, c_rarg2, c_rarg3); 829 assert_different_registers(arg_1, c_rarg2, c_rarg3); 830 assert_different_registers(arg_2, c_rarg3); 831 832 pass_arg3(this, arg_3); 833 pass_arg2(this, arg_2); 834 pass_arg1(this, arg_1); 835 pass_arg0(this, arg_0); 836 MacroAssembler::call_VM_leaf_base(entry_point, 4); 837 } 838 839 void MacroAssembler::la(Register Rd, const address addr) { 840 int32_t offset; 841 la(Rd, addr, offset); 842 addi(Rd, Rd, offset); 843 } 844 845 void MacroAssembler::la(Register Rd, const address addr, int32_t &offset) { 846 if (is_32bit_offset_from_codecache((int64_t)addr)) { 847 int64_t distance = addr - pc(); 848 assert(is_valid_32bit_offset(distance), "Must be"); 849 auipc(Rd, (int32_t)distance + 0x800); 850 offset = ((int32_t)distance << 20) >> 20; 851 } else { 852 assert(!CodeCache::contains(addr), "Must be"); 853 movptr(Rd, addr, offset); 854 } 855 } 856 857 void MacroAssembler::la(Register Rd, const Address &adr) { 858 switch (adr.getMode()) { 859 case Address::literal: { 860 relocInfo::relocType rtype = adr.rspec().reloc()->type(); 861 if (rtype == relocInfo::none) { 862 mv(Rd, (intptr_t)(adr.target())); 863 } else { 864 relocate(adr.rspec(), [&] { 865 movptr(Rd, adr.target()); 866 }); 867 } 868 break; 869 } 870 case Address::base_plus_offset: { 871 Address new_adr = legitimize_address(Rd, adr); 872 if (!(new_adr.base() == Rd && new_adr.offset() == 0)) { 873 addi(Rd, new_adr.base(), new_adr.offset()); 874 } 875 break; 876 } 877 default: 878 ShouldNotReachHere(); 879 } 880 } 881 882 void MacroAssembler::la(Register Rd, Label &label) { 883 IncompressibleRegion ir(this); // the label address may be patched back. 884 wrap_label(Rd, label, &MacroAssembler::la); 885 } 886 887 void MacroAssembler::li16u(Register Rd, uint16_t imm) { 888 lui(Rd, (uint32_t)imm << 12); 889 srli(Rd, Rd, 12); 890 } 891 892 void MacroAssembler::li32(Register Rd, int32_t imm) { 893 // int32_t is in range 0x8000 0000 ~ 0x7fff ffff, and imm[31] is the sign bit 894 int64_t upper = imm, lower = imm; 895 lower = (imm << 20) >> 20; 896 upper -= lower; 897 upper = (int32_t)upper; 898 // lui Rd, imm[31:12] + imm[11] 899 lui(Rd, upper); 900 addiw(Rd, Rd, lower); 901 } 902 903 void MacroAssembler::li(Register Rd, int64_t imm) { 904 // int64_t is in range 0x8000 0000 0000 0000 ~ 0x7fff ffff ffff ffff 905 // li -> c.li 906 if (do_compress() && (is_simm6(imm) && Rd != x0)) { 907 c_li(Rd, imm); 908 return; 909 } 910 911 int shift = 12; 912 int64_t upper = imm, lower = imm; 913 // Split imm to a lower 12-bit sign-extended part and the remainder, 914 // because addi will sign-extend the lower imm. 915 lower = ((int32_t)imm << 20) >> 20; 916 upper -= lower; 917 918 // Test whether imm is a 32-bit integer. 919 if (!(((imm) & ~(int64_t)0x7fffffff) == 0 || 920 (((imm) & ~(int64_t)0x7fffffff) == ~(int64_t)0x7fffffff))) { 921 while (((upper >> shift) & 1) == 0) { shift++; } 922 upper >>= shift; 923 li(Rd, upper); 924 slli(Rd, Rd, shift); 925 if (lower != 0) { 926 addi(Rd, Rd, lower); 927 } 928 } else { 929 // 32-bit integer 930 Register hi_Rd = zr; 931 if (upper != 0) { 932 lui(Rd, (int32_t)upper); 933 hi_Rd = Rd; 934 } 935 if (lower != 0 || hi_Rd == zr) { 936 addiw(Rd, hi_Rd, lower); 937 } 938 } 939 } 940 941 void MacroAssembler::load_link_jump(const address source, Register temp) { 942 assert(temp != noreg && temp != x0, "expecting a register"); 943 assert_cond(source != nullptr); 944 int64_t distance = source - pc(); 945 assert(is_simm32(distance), "Must be"); 946 auipc(temp, (int32_t)distance + 0x800); 947 ld(temp, Address(temp, ((int32_t)distance << 20) >> 20)); 948 jalr(temp); 949 } 950 951 void MacroAssembler::jump_link(const address dest, Register temp) { 952 assert(UseTrampolines, "Must be"); 953 assert_cond(dest != nullptr); 954 int64_t distance = dest - pc(); 955 assert(is_simm21(distance), "Must be"); 956 assert((distance % 2) == 0, "Must be"); 957 jal(x1, distance); 958 } 959 960 void MacroAssembler::j(const address dest, Register temp) { 961 assert(CodeCache::contains(dest), "Must be"); 962 assert_cond(dest != nullptr); 963 int64_t distance = dest - pc(); 964 965 // We can't patch C, i.e. if Label wasn't bound we need to patch this jump. 966 IncompressibleRegion ir(this); 967 if (is_simm21(distance) && ((distance % 2) == 0)) { 968 Assembler::jal(x0, distance); 969 } else { 970 assert(temp != noreg && temp != x0, "expecting a register"); 971 int32_t offset = 0; 972 la(temp, dest, offset); 973 jr(temp, offset); 974 } 975 } 976 977 void MacroAssembler::j(const Address &adr, Register temp) { 978 switch (adr.getMode()) { 979 case Address::literal: { 980 relocate(adr.rspec(), [&] { 981 j(adr.target(), temp); 982 }); 983 break; 984 } 985 case Address::base_plus_offset: { 986 int32_t offset = ((int32_t)adr.offset() << 20) >> 20; 987 la(temp, Address(adr.base(), adr.offset() - offset)); 988 jr(temp, offset); 989 break; 990 } 991 default: 992 ShouldNotReachHere(); 993 } 994 } 995 996 void MacroAssembler::j(Label &lab, Register temp) { 997 assert_different_registers(x0, temp); 998 if (lab.is_bound()) { 999 MacroAssembler::j(target(lab), temp); 1000 } else { 1001 lab.add_patch_at(code(), locator()); 1002 MacroAssembler::j(pc(), temp); 1003 } 1004 } 1005 1006 void MacroAssembler::jr(Register Rd, int32_t offset) { 1007 assert(Rd != noreg, "expecting a register"); 1008 Assembler::jalr(x0, Rd, offset); 1009 } 1010 1011 void MacroAssembler::call(const address dest, Register temp) { 1012 assert_cond(dest != nullptr); 1013 assert(temp != noreg, "expecting a register"); 1014 int32_t offset = 0; 1015 la(temp, dest, offset); 1016 jalr(temp, offset); 1017 } 1018 1019 void MacroAssembler::jalr(Register Rs, int32_t offset) { 1020 assert(Rs != noreg, "expecting a register"); 1021 Assembler::jalr(x1, Rs, offset); 1022 } 1023 1024 void MacroAssembler::rt_call(address dest, Register tmp) { 1025 CodeBlob *cb = CodeCache::find_blob(dest); 1026 RuntimeAddress target(dest); 1027 if (cb) { 1028 far_call(target, tmp); 1029 } else { 1030 relocate(target.rspec(), [&] { 1031 int32_t offset; 1032 la(tmp, target.target(), offset); 1033 jalr(tmp, offset); 1034 }); 1035 } 1036 } 1037 1038 void MacroAssembler::wrap_label(Register Rt, Label &L, jal_jalr_insn insn) { 1039 if (L.is_bound()) { 1040 (this->*insn)(Rt, target(L)); 1041 } else { 1042 L.add_patch_at(code(), locator()); 1043 (this->*insn)(Rt, pc()); 1044 } 1045 } 1046 1047 void MacroAssembler::wrap_label(Register r1, Register r2, Label &L, 1048 compare_and_branch_insn insn, 1049 compare_and_branch_label_insn neg_insn, bool is_far) { 1050 if (is_far) { 1051 Label done; 1052 (this->*neg_insn)(r1, r2, done, /* is_far */ false); 1053 j(L); 1054 bind(done); 1055 } else { 1056 if (L.is_bound()) { 1057 (this->*insn)(r1, r2, target(L)); 1058 } else { 1059 L.add_patch_at(code(), locator()); 1060 (this->*insn)(r1, r2, pc()); 1061 } 1062 } 1063 } 1064 1065 #define INSN(NAME, NEG_INSN) \ 1066 void MacroAssembler::NAME(Register Rs1, Register Rs2, Label &L, bool is_far) { \ 1067 wrap_label(Rs1, Rs2, L, &MacroAssembler::NAME, &MacroAssembler::NEG_INSN, is_far); \ 1068 } 1069 1070 INSN(beq, bne); 1071 INSN(bne, beq); 1072 INSN(blt, bge); 1073 INSN(bge, blt); 1074 INSN(bltu, bgeu); 1075 INSN(bgeu, bltu); 1076 1077 #undef INSN 1078 1079 #define INSN(NAME) \ 1080 void MacroAssembler::NAME##z(Register Rs, const address dest) { \ 1081 NAME(Rs, zr, dest); \ 1082 } \ 1083 void MacroAssembler::NAME##z(Register Rs, Label &l, bool is_far) { \ 1084 NAME(Rs, zr, l, is_far); \ 1085 } \ 1086 1087 INSN(beq); 1088 INSN(bne); 1089 INSN(blt); 1090 INSN(ble); 1091 INSN(bge); 1092 INSN(bgt); 1093 1094 #undef INSN 1095 1096 #define INSN(NAME, NEG_INSN) \ 1097 void MacroAssembler::NAME(Register Rs, Register Rt, const address dest) { \ 1098 NEG_INSN(Rt, Rs, dest); \ 1099 } \ 1100 void MacroAssembler::NAME(Register Rs, Register Rt, Label &l, bool is_far) { \ 1101 NEG_INSN(Rt, Rs, l, is_far); \ 1102 } 1103 1104 INSN(bgt, blt); 1105 INSN(ble, bge); 1106 INSN(bgtu, bltu); 1107 INSN(bleu, bgeu); 1108 1109 #undef INSN 1110 1111 // Float compare branch instructions 1112 1113 #define INSN(NAME, FLOATCMP, BRANCH) \ 1114 void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far, bool is_unordered) { \ 1115 FLOATCMP##_s(t0, Rs1, Rs2); \ 1116 BRANCH(t0, l, is_far); \ 1117 } \ 1118 void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far, bool is_unordered) { \ 1119 FLOATCMP##_d(t0, Rs1, Rs2); \ 1120 BRANCH(t0, l, is_far); \ 1121 } 1122 1123 INSN(beq, feq, bnez); 1124 INSN(bne, feq, beqz); 1125 1126 #undef INSN 1127 1128 1129 #define INSN(NAME, FLOATCMP1, FLOATCMP2) \ 1130 void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ 1131 bool is_far, bool is_unordered) { \ 1132 if (is_unordered) { \ 1133 /* jump if either source is NaN or condition is expected */ \ 1134 FLOATCMP2##_s(t0, Rs2, Rs1); \ 1135 beqz(t0, l, is_far); \ 1136 } else { \ 1137 /* jump if no NaN in source and condition is expected */ \ 1138 FLOATCMP1##_s(t0, Rs1, Rs2); \ 1139 bnez(t0, l, is_far); \ 1140 } \ 1141 } \ 1142 void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ 1143 bool is_far, bool is_unordered) { \ 1144 if (is_unordered) { \ 1145 /* jump if either source is NaN or condition is expected */ \ 1146 FLOATCMP2##_d(t0, Rs2, Rs1); \ 1147 beqz(t0, l, is_far); \ 1148 } else { \ 1149 /* jump if no NaN in source and condition is expected */ \ 1150 FLOATCMP1##_d(t0, Rs1, Rs2); \ 1151 bnez(t0, l, is_far); \ 1152 } \ 1153 } 1154 1155 INSN(ble, fle, flt); 1156 INSN(blt, flt, fle); 1157 1158 #undef INSN 1159 1160 #define INSN(NAME, CMP) \ 1161 void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ 1162 bool is_far, bool is_unordered) { \ 1163 float_##CMP(Rs2, Rs1, l, is_far, is_unordered); \ 1164 } \ 1165 void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ 1166 bool is_far, bool is_unordered) { \ 1167 double_##CMP(Rs2, Rs1, l, is_far, is_unordered); \ 1168 } 1169 1170 INSN(bgt, blt); 1171 INSN(bge, ble); 1172 1173 #undef INSN 1174 1175 1176 #define INSN(NAME, CSR) \ 1177 void MacroAssembler::NAME(Register Rd) { \ 1178 csrr(Rd, CSR); \ 1179 } 1180 1181 INSN(rdinstret, CSR_INSTRET); 1182 INSN(rdcycle, CSR_CYCLE); 1183 INSN(rdtime, CSR_TIME); 1184 INSN(frcsr, CSR_FCSR); 1185 INSN(frrm, CSR_FRM); 1186 INSN(frflags, CSR_FFLAGS); 1187 1188 #undef INSN 1189 1190 void MacroAssembler::csrr(Register Rd, unsigned csr) { 1191 csrrs(Rd, csr, x0); 1192 } 1193 1194 #define INSN(NAME, OPFUN) \ 1195 void MacroAssembler::NAME(unsigned csr, Register Rs) { \ 1196 OPFUN(x0, csr, Rs); \ 1197 } 1198 1199 INSN(csrw, csrrw); 1200 INSN(csrs, csrrs); 1201 INSN(csrc, csrrc); 1202 1203 #undef INSN 1204 1205 #define INSN(NAME, OPFUN) \ 1206 void MacroAssembler::NAME(unsigned csr, unsigned imm) { \ 1207 OPFUN(x0, csr, imm); \ 1208 } 1209 1210 INSN(csrwi, csrrwi); 1211 INSN(csrsi, csrrsi); 1212 INSN(csrci, csrrci); 1213 1214 #undef INSN 1215 1216 #define INSN(NAME, CSR) \ 1217 void MacroAssembler::NAME(Register Rd, Register Rs) { \ 1218 csrrw(Rd, CSR, Rs); \ 1219 } 1220 1221 INSN(fscsr, CSR_FCSR); 1222 INSN(fsrm, CSR_FRM); 1223 INSN(fsflags, CSR_FFLAGS); 1224 1225 #undef INSN 1226 1227 #define INSN(NAME) \ 1228 void MacroAssembler::NAME(Register Rs) { \ 1229 NAME(x0, Rs); \ 1230 } 1231 1232 INSN(fscsr); 1233 INSN(fsrm); 1234 INSN(fsflags); 1235 1236 #undef INSN 1237 1238 void MacroAssembler::fsrmi(Register Rd, unsigned imm) { 1239 guarantee(imm < 5, "Rounding Mode is invalid in Rounding Mode register"); 1240 csrrwi(Rd, CSR_FRM, imm); 1241 } 1242 1243 void MacroAssembler::fsflagsi(Register Rd, unsigned imm) { 1244 csrrwi(Rd, CSR_FFLAGS, imm); 1245 } 1246 1247 #define INSN(NAME) \ 1248 void MacroAssembler::NAME(unsigned imm) { \ 1249 NAME(x0, imm); \ 1250 } 1251 1252 INSN(fsrmi); 1253 INSN(fsflagsi); 1254 1255 #undef INSN 1256 1257 void MacroAssembler::restore_cpu_control_state_after_jni(Register tmp) { 1258 if (RestoreMXCSROnJNICalls) { 1259 Label skip_fsrmi; 1260 frrm(tmp); 1261 // Set FRM to the state we need. We do want Round to Nearest. 1262 // We don't want non-IEEE rounding modes. 1263 guarantee(RoundingMode::rne == 0, "must be"); 1264 beqz(tmp, skip_fsrmi); // Only reset FRM if it's wrong 1265 fsrmi(RoundingMode::rne); 1266 bind(skip_fsrmi); 1267 } 1268 } 1269 1270 void MacroAssembler::push_reg(Register Rs) 1271 { 1272 addi(esp, esp, 0 - wordSize); 1273 sd(Rs, Address(esp, 0)); 1274 } 1275 1276 void MacroAssembler::pop_reg(Register Rd) 1277 { 1278 ld(Rd, Address(esp, 0)); 1279 addi(esp, esp, wordSize); 1280 } 1281 1282 int MacroAssembler::bitset_to_regs(unsigned int bitset, unsigned char* regs) { 1283 int count = 0; 1284 // Scan bitset to accumulate register pairs 1285 for (int reg = 31; reg >= 0; reg--) { 1286 if ((1U << 31) & bitset) { 1287 regs[count++] = reg; 1288 } 1289 bitset <<= 1; 1290 } 1291 return count; 1292 } 1293 1294 // Push integer registers in the bitset supplied. Don't push sp. 1295 // Return the number of words pushed 1296 int MacroAssembler::push_reg(unsigned int bitset, Register stack) { 1297 DEBUG_ONLY(int words_pushed = 0;) 1298 unsigned char regs[32]; 1299 int count = bitset_to_regs(bitset, regs); 1300 // reserve one slot to align for odd count 1301 int offset = is_even(count) ? 0 : wordSize; 1302 1303 if (count) { 1304 addi(stack, stack, -count * wordSize - offset); 1305 } 1306 for (int i = count - 1; i >= 0; i--) { 1307 sd(as_Register(regs[i]), Address(stack, (count - 1 - i) * wordSize + offset)); 1308 DEBUG_ONLY(words_pushed++;) 1309 } 1310 1311 assert(words_pushed == count, "oops, pushed != count"); 1312 1313 return count; 1314 } 1315 1316 int MacroAssembler::pop_reg(unsigned int bitset, Register stack) { 1317 DEBUG_ONLY(int words_popped = 0;) 1318 unsigned char regs[32]; 1319 int count = bitset_to_regs(bitset, regs); 1320 // reserve one slot to align for odd count 1321 int offset = is_even(count) ? 0 : wordSize; 1322 1323 for (int i = count - 1; i >= 0; i--) { 1324 ld(as_Register(regs[i]), Address(stack, (count - 1 - i) * wordSize + offset)); 1325 DEBUG_ONLY(words_popped++;) 1326 } 1327 1328 if (count) { 1329 addi(stack, stack, count * wordSize + offset); 1330 } 1331 assert(words_popped == count, "oops, popped != count"); 1332 1333 return count; 1334 } 1335 1336 // Push floating-point registers in the bitset supplied. 1337 // Return the number of words pushed 1338 int MacroAssembler::push_fp(unsigned int bitset, Register stack) { 1339 DEBUG_ONLY(int words_pushed = 0;) 1340 unsigned char regs[32]; 1341 int count = bitset_to_regs(bitset, regs); 1342 int push_slots = count + (count & 1); 1343 1344 if (count) { 1345 addi(stack, stack, -push_slots * wordSize); 1346 } 1347 1348 for (int i = count - 1; i >= 0; i--) { 1349 fsd(as_FloatRegister(regs[i]), Address(stack, (push_slots - 1 - i) * wordSize)); 1350 DEBUG_ONLY(words_pushed++;) 1351 } 1352 1353 assert(words_pushed == count, "oops, pushed(%d) != count(%d)", words_pushed, count); 1354 1355 return count; 1356 } 1357 1358 int MacroAssembler::pop_fp(unsigned int bitset, Register stack) { 1359 DEBUG_ONLY(int words_popped = 0;) 1360 unsigned char regs[32]; 1361 int count = bitset_to_regs(bitset, regs); 1362 int pop_slots = count + (count & 1); 1363 1364 for (int i = count - 1; i >= 0; i--) { 1365 fld(as_FloatRegister(regs[i]), Address(stack, (pop_slots - 1 - i) * wordSize)); 1366 DEBUG_ONLY(words_popped++;) 1367 } 1368 1369 if (count) { 1370 addi(stack, stack, pop_slots * wordSize); 1371 } 1372 1373 assert(words_popped == count, "oops, popped(%d) != count(%d)", words_popped, count); 1374 1375 return count; 1376 } 1377 1378 static const int64_t right_32_bits = right_n_bits(32); 1379 static const int64_t right_8_bits = right_n_bits(8); 1380 1381 /** 1382 * Emits code to update CRC-32 with a byte value according to constants in table 1383 * 1384 * @param [in,out]crc Register containing the crc. 1385 * @param [in]val Register containing the byte to fold into the CRC. 1386 * @param [in]table Register containing the table of crc constants. 1387 * 1388 * uint32_t crc; 1389 * val = crc_table[(val ^ crc) & 0xFF]; 1390 * crc = val ^ (crc >> 8); 1391 * 1392 */ 1393 void MacroAssembler::update_byte_crc32(Register crc, Register val, Register table) { 1394 assert_different_registers(crc, val, table); 1395 1396 xorr(val, val, crc); 1397 andi(val, val, right_8_bits); 1398 shadd(val, val, table, val, 2); 1399 lwu(val, Address(val)); 1400 srli(crc, crc, 8); 1401 xorr(crc, val, crc); 1402 } 1403 1404 /** 1405 * Emits code to update CRC-32 with a 32-bit value according to tables 0 to 3 1406 * 1407 * @param [in,out]crc Register containing the crc. 1408 * @param [in]v Register containing the 32-bit to fold into the CRC. 1409 * @param [in]table0 Register containing table 0 of crc constants. 1410 * @param [in]table1 Register containing table 1 of crc constants. 1411 * @param [in]table2 Register containing table 2 of crc constants. 1412 * @param [in]table3 Register containing table 3 of crc constants. 1413 * 1414 * uint32_t crc; 1415 * v = crc ^ v 1416 * crc = table3[v&0xff]^table2[(v>>8)&0xff]^table1[(v>>16)&0xff]^table0[v>>24] 1417 * 1418 */ 1419 void MacroAssembler::update_word_crc32(Register crc, Register v, Register tmp1, Register tmp2, Register tmp3, 1420 Register table0, Register table1, Register table2, Register table3, bool upper) { 1421 assert_different_registers(crc, v, tmp1, tmp2, tmp3, table0, table1, table2, table3); 1422 1423 if (upper) 1424 srli(v, v, 32); 1425 xorr(v, v, crc); 1426 1427 andi(tmp1, v, right_8_bits); 1428 shadd(tmp1, tmp1, table3, tmp2, 2); 1429 lwu(crc, Address(tmp1)); 1430 1431 slli(tmp1, v, 16); 1432 slli(tmp3, v, 8); 1433 1434 srliw(tmp1, tmp1, 24); 1435 srliw(tmp3, tmp3, 24); 1436 1437 shadd(tmp1, tmp1, table2, tmp1, 2); 1438 lwu(tmp2, Address(tmp1)); 1439 1440 shadd(tmp3, tmp3, table1, tmp3, 2); 1441 xorr(crc, crc, tmp2); 1442 1443 lwu(tmp2, Address(tmp3)); 1444 // It is more optimal to use 'srli' instead of 'srliw' for case when it is not necessary to clean upper bits 1445 if (upper) 1446 srli(tmp1, v, 24); 1447 else 1448 srliw(tmp1, v, 24); 1449 1450 // no need to clear bits other than lowest two 1451 shadd(tmp1, tmp1, table0, tmp1, 2); 1452 xorr(crc, crc, tmp2); 1453 lwu(tmp2, Address(tmp1)); 1454 xorr(crc, crc, tmp2); 1455 } 1456 1457 /** 1458 * @param crc register containing existing CRC (32-bit) 1459 * @param buf register pointing to input byte buffer (byte*) 1460 * @param len register containing number of bytes 1461 * @param table register that will contain address of CRC table 1462 * @param tmp scratch registers 1463 */ 1464 void MacroAssembler::kernel_crc32(Register crc, Register buf, Register len, 1465 Register table0, Register table1, Register table2, Register table3, 1466 Register tmp1, Register tmp2, Register tmp3, Register tmp4, Register tmp5, Register tmp6) { 1467 assert_different_registers(crc, buf, len, table0, table1, table2, table3, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6); 1468 Label L_by16_loop, L_unroll_loop, L_unroll_loop_entry, L_by4, L_by4_loop, L_by1, L_by1_loop, L_exit; 1469 1470 const int64_t unroll = 16; 1471 const int64_t unroll_words = unroll*wordSize; 1472 mv(tmp5, right_32_bits); 1473 subw(len, len, unroll_words); 1474 andn(crc, tmp5, crc); 1475 1476 const ExternalAddress table_addr = StubRoutines::crc_table_addr(); 1477 la(table0, table_addr); 1478 add(table1, table0, 1*256*sizeof(juint), tmp1); 1479 add(table2, table0, 2*256*sizeof(juint), tmp1); 1480 add(table3, table2, 1*256*sizeof(juint), tmp1); 1481 1482 bge(len, zr, L_unroll_loop_entry); 1483 addiw(len, len, unroll_words-4); 1484 bge(len, zr, L_by4_loop); 1485 addiw(len, len, 4); 1486 bgt(len, zr, L_by1_loop); 1487 j(L_exit); 1488 1489 align(CodeEntryAlignment); 1490 bind(L_unroll_loop_entry); 1491 const Register buf_end = tmp3; 1492 add(buf_end, buf, len); // buf_end will be used as endpoint for loop below 1493 andi(len, len, unroll_words-1); // len = (len % unroll_words) 1494 sub(len, len, unroll_words); // Length after all iterations 1495 bind(L_unroll_loop); 1496 for (int i = 0; i < unroll; i++) { 1497 ld(tmp1, Address(buf, i*wordSize)); 1498 update_word_crc32(crc, tmp1, tmp2, tmp4, tmp6, table0, table1, table2, table3, false); 1499 update_word_crc32(crc, tmp1, tmp2, tmp4, tmp6, table0, table1, table2, table3, true); 1500 } 1501 1502 addi(buf, buf, unroll_words); 1503 ble(buf, buf_end, L_unroll_loop); 1504 addiw(len, len, unroll_words-4); 1505 bge(len, zr, L_by4_loop); 1506 addiw(len, len, 4); 1507 bgt(len, zr, L_by1_loop); 1508 j(L_exit); 1509 1510 bind(L_by4_loop); 1511 lwu(tmp1, Address(buf)); 1512 update_word_crc32(crc, tmp1, tmp2, tmp4, tmp6, table0, table1, table2, table3, false); 1513 subw(len, len, 4); 1514 addi(buf, buf, 4); 1515 bge(len, zr, L_by4_loop); 1516 addiw(len, len, 4); 1517 ble(len, zr, L_exit); 1518 1519 bind(L_by1_loop); 1520 subw(len, len, 1); 1521 lwu(tmp1, Address(buf)); 1522 andi(tmp2, tmp1, right_8_bits); 1523 update_byte_crc32(crc, tmp2, table0); 1524 ble(len, zr, L_exit); 1525 1526 subw(len, len, 1); 1527 srli(tmp2, tmp1, 8); 1528 andi(tmp2, tmp2, right_8_bits); 1529 update_byte_crc32(crc, tmp2, table0); 1530 ble(len, zr, L_exit); 1531 1532 subw(len, len, 1); 1533 srli(tmp2, tmp1, 16); 1534 andi(tmp2, tmp2, right_8_bits); 1535 update_byte_crc32(crc, tmp2, table0); 1536 ble(len, zr, L_exit); 1537 1538 srli(tmp2, tmp1, 24); 1539 andi(tmp2, tmp2, right_8_bits); 1540 update_byte_crc32(crc, tmp2, table0); 1541 1542 bind(L_exit); 1543 andn(crc, tmp5, crc); 1544 } 1545 1546 #ifdef COMPILER2 1547 // Push vector registers in the bitset supplied. 1548 // Return the number of words pushed 1549 int MacroAssembler::push_v(unsigned int bitset, Register stack) { 1550 int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE); 1551 1552 // Scan bitset to accumulate register pairs 1553 unsigned char regs[32]; 1554 int count = bitset_to_regs(bitset, regs); 1555 1556 for (int i = 0; i < count; i++) { 1557 sub(stack, stack, vector_size_in_bytes); 1558 vs1r_v(as_VectorRegister(regs[i]), stack); 1559 } 1560 1561 return count * vector_size_in_bytes / wordSize; 1562 } 1563 1564 int MacroAssembler::pop_v(unsigned int bitset, Register stack) { 1565 int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE); 1566 1567 // Scan bitset to accumulate register pairs 1568 unsigned char regs[32]; 1569 int count = bitset_to_regs(bitset, regs); 1570 1571 for (int i = count - 1; i >= 0; i--) { 1572 vl1r_v(as_VectorRegister(regs[i]), stack); 1573 add(stack, stack, vector_size_in_bytes); 1574 } 1575 1576 return count * vector_size_in_bytes / wordSize; 1577 } 1578 #endif // COMPILER2 1579 1580 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude) { 1581 // Push integer registers x7, x10-x17, x28-x31. 1582 push_reg(RegSet::of(x7) + RegSet::range(x10, x17) + RegSet::range(x28, x31) - exclude, sp); 1583 1584 // Push float registers f0-f7, f10-f17, f28-f31. 1585 addi(sp, sp, - wordSize * 20); 1586 int offset = 0; 1587 for (int i = 0; i < 32; i++) { 1588 if (i <= f7->encoding() || i >= f28->encoding() || (i >= f10->encoding() && i <= f17->encoding())) { 1589 fsd(as_FloatRegister(i), Address(sp, wordSize * (offset++))); 1590 } 1591 } 1592 } 1593 1594 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) { 1595 int offset = 0; 1596 for (int i = 0; i < 32; i++) { 1597 if (i <= f7->encoding() || i >= f28->encoding() || (i >= f10->encoding() && i <= f17->encoding())) { 1598 fld(as_FloatRegister(i), Address(sp, wordSize * (offset++))); 1599 } 1600 } 1601 addi(sp, sp, wordSize * 20); 1602 1603 pop_reg(RegSet::of(x7) + RegSet::range(x10, x17) + RegSet::range(x28, x31) - exclude, sp); 1604 } 1605 1606 void MacroAssembler::push_CPU_state(bool save_vectors, int vector_size_in_bytes) { 1607 // integer registers, except zr(x0) & ra(x1) & sp(x2) & gp(x3) & tp(x4) 1608 push_reg(RegSet::range(x5, x31), sp); 1609 1610 // float registers 1611 addi(sp, sp, - 32 * wordSize); 1612 for (int i = 0; i < 32; i++) { 1613 fsd(as_FloatRegister(i), Address(sp, i * wordSize)); 1614 } 1615 1616 // vector registers 1617 if (save_vectors) { 1618 sub(sp, sp, vector_size_in_bytes * VectorRegister::number_of_registers); 1619 vsetvli(t0, x0, Assembler::e64, Assembler::m8); 1620 for (int i = 0; i < VectorRegister::number_of_registers; i += 8) { 1621 add(t0, sp, vector_size_in_bytes * i); 1622 vse64_v(as_VectorRegister(i), t0); 1623 } 1624 } 1625 } 1626 1627 void MacroAssembler::pop_CPU_state(bool restore_vectors, int vector_size_in_bytes) { 1628 // vector registers 1629 if (restore_vectors) { 1630 vsetvli(t0, x0, Assembler::e64, Assembler::m8); 1631 for (int i = 0; i < VectorRegister::number_of_registers; i += 8) { 1632 vle64_v(as_VectorRegister(i), sp); 1633 add(sp, sp, vector_size_in_bytes * 8); 1634 } 1635 } 1636 1637 // float registers 1638 for (int i = 0; i < 32; i++) { 1639 fld(as_FloatRegister(i), Address(sp, i * wordSize)); 1640 } 1641 addi(sp, sp, 32 * wordSize); 1642 1643 // integer registers, except zr(x0) & ra(x1) & sp(x2) & gp(x3) & tp(x4) 1644 pop_reg(RegSet::range(x5, x31), sp); 1645 } 1646 1647 static int patch_offset_in_jal(address branch, int64_t offset) { 1648 assert(Assembler::is_simm21(offset) && ((offset % 2) == 0), 1649 "offset is too large to be patched in one jal instruction!\n"); 1650 Assembler::patch(branch, 31, 31, (offset >> 20) & 0x1); // offset[20] ==> branch[31] 1651 Assembler::patch(branch, 30, 21, (offset >> 1) & 0x3ff); // offset[10:1] ==> branch[30:21] 1652 Assembler::patch(branch, 20, 20, (offset >> 11) & 0x1); // offset[11] ==> branch[20] 1653 Assembler::patch(branch, 19, 12, (offset >> 12) & 0xff); // offset[19:12] ==> branch[19:12] 1654 return MacroAssembler::instruction_size; // only one instruction 1655 } 1656 1657 static int patch_offset_in_conditional_branch(address branch, int64_t offset) { 1658 assert(Assembler::is_simm13(offset) && ((offset % 2) == 0), 1659 "offset is too large to be patched in one beq/bge/bgeu/blt/bltu/bne instruction!\n"); 1660 Assembler::patch(branch, 31, 31, (offset >> 12) & 0x1); // offset[12] ==> branch[31] 1661 Assembler::patch(branch, 30, 25, (offset >> 5) & 0x3f); // offset[10:5] ==> branch[30:25] 1662 Assembler::patch(branch, 7, 7, (offset >> 11) & 0x1); // offset[11] ==> branch[7] 1663 Assembler::patch(branch, 11, 8, (offset >> 1) & 0xf); // offset[4:1] ==> branch[11:8] 1664 return MacroAssembler::instruction_size; // only one instruction 1665 } 1666 1667 static int patch_offset_in_pc_relative(address branch, int64_t offset) { 1668 const int PC_RELATIVE_INSTRUCTION_NUM = 2; // auipc, addi/jalr/load 1669 Assembler::patch(branch, 31, 12, ((offset + 0x800) >> 12) & 0xfffff); // Auipc. offset[31:12] ==> branch[31:12] 1670 Assembler::patch(branch + 4, 31, 20, offset & 0xfff); // Addi/Jalr/Load. offset[11:0] ==> branch[31:20] 1671 return PC_RELATIVE_INSTRUCTION_NUM * MacroAssembler::instruction_size; 1672 } 1673 1674 static int patch_addr_in_movptr1(address branch, address target) { 1675 int32_t lower = ((intptr_t)target << 35) >> 35; 1676 int64_t upper = ((intptr_t)target - lower) >> 29; 1677 Assembler::patch(branch + 0, 31, 12, upper & 0xfffff); // Lui. target[48:29] + target[28] ==> branch[31:12] 1678 Assembler::patch(branch + 4, 31, 20, (lower >> 17) & 0xfff); // Addi. target[28:17] ==> branch[31:20] 1679 Assembler::patch(branch + 12, 31, 20, (lower >> 6) & 0x7ff); // Addi. target[16: 6] ==> branch[31:20] 1680 Assembler::patch(branch + 20, 31, 20, lower & 0x3f); // Addi/Jalr/Load. target[ 5: 0] ==> branch[31:20] 1681 return MacroAssembler::movptr1_instruction_size; 1682 } 1683 1684 static int patch_addr_in_movptr2(address instruction_address, address target) { 1685 uintptr_t addr = (uintptr_t)target; 1686 1687 assert(addr < (1ull << 48), "48-bit overflow in address constant"); 1688 unsigned int upper18 = (addr >> 30ull); 1689 int lower30 = (addr & 0x3fffffffu); 1690 int low12 = (lower30 << 20) >> 20; 1691 int mid18 = ((lower30 - low12) >> 12); 1692 1693 Assembler::patch(instruction_address + (MacroAssembler::instruction_size * 0), 31, 12, (upper18 & 0xfffff)); // Lui 1694 Assembler::patch(instruction_address + (MacroAssembler::instruction_size * 1), 31, 12, (mid18 & 0xfffff)); // Lui 1695 // Slli 1696 // Add 1697 Assembler::patch(instruction_address + (MacroAssembler::instruction_size * 4), 31, 20, low12 & 0xfff); // Addi/Jalr/Load 1698 1699 assert(MacroAssembler::target_addr_for_insn(instruction_address) == target, "Must be"); 1700 1701 return MacroAssembler::movptr2_instruction_size; 1702 } 1703 1704 static int patch_imm_in_li16u(address branch, uint16_t target) { 1705 Assembler::patch(branch, 31, 12, target); // patch lui only 1706 return MacroAssembler::instruction_size; 1707 } 1708 1709 int MacroAssembler::patch_imm_in_li32(address branch, int32_t target) { 1710 const int LI32_INSTRUCTIONS_NUM = 2; // lui + addiw 1711 int64_t upper = (intptr_t)target; 1712 int32_t lower = (((int32_t)target) << 20) >> 20; 1713 upper -= lower; 1714 upper = (int32_t)upper; 1715 Assembler::patch(branch + 0, 31, 12, (upper >> 12) & 0xfffff); // Lui. 1716 Assembler::patch(branch + 4, 31, 20, lower & 0xfff); // Addiw. 1717 return LI32_INSTRUCTIONS_NUM * MacroAssembler::instruction_size; 1718 } 1719 1720 static long get_offset_of_jal(address insn_addr) { 1721 assert_cond(insn_addr != nullptr); 1722 long offset = 0; 1723 unsigned insn = Assembler::ld_instr(insn_addr); 1724 long val = (long)Assembler::sextract(insn, 31, 12); 1725 offset |= ((val >> 19) & 0x1) << 20; 1726 offset |= (val & 0xff) << 12; 1727 offset |= ((val >> 8) & 0x1) << 11; 1728 offset |= ((val >> 9) & 0x3ff) << 1; 1729 offset = (offset << 43) >> 43; 1730 return offset; 1731 } 1732 1733 static long get_offset_of_conditional_branch(address insn_addr) { 1734 long offset = 0; 1735 assert_cond(insn_addr != nullptr); 1736 unsigned insn = Assembler::ld_instr(insn_addr); 1737 offset = (long)Assembler::sextract(insn, 31, 31); 1738 offset = (offset << 12) | (((long)(Assembler::sextract(insn, 7, 7) & 0x1)) << 11); 1739 offset = offset | (((long)(Assembler::sextract(insn, 30, 25) & 0x3f)) << 5); 1740 offset = offset | (((long)(Assembler::sextract(insn, 11, 8) & 0xf)) << 1); 1741 offset = (offset << 41) >> 41; 1742 return offset; 1743 } 1744 1745 static long get_offset_of_pc_relative(address insn_addr) { 1746 long offset = 0; 1747 assert_cond(insn_addr != nullptr); 1748 offset = ((long)(Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12))) << 12; // Auipc. 1749 offset += ((long)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20)); // Addi/Jalr/Load. 1750 offset = (offset << 32) >> 32; 1751 return offset; 1752 } 1753 1754 static address get_target_of_movptr1(address insn_addr) { 1755 assert_cond(insn_addr != nullptr); 1756 intptr_t target_address = (((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12)) & 0xfffff) << 29; // Lui. 1757 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20)) << 17; // Addi. 1758 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 12), 31, 20)) << 6; // Addi. 1759 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 20), 31, 20)); // Addi/Jalr/Load. 1760 return (address) target_address; 1761 } 1762 1763 static address get_target_of_movptr2(address insn_addr) { 1764 assert_cond(insn_addr != nullptr); 1765 int32_t upper18 = ((Assembler::sextract(Assembler::ld_instr(insn_addr + MacroAssembler::instruction_size * 0), 31, 12)) & 0xfffff); // Lui 1766 int32_t mid18 = ((Assembler::sextract(Assembler::ld_instr(insn_addr + MacroAssembler::instruction_size * 1), 31, 12)) & 0xfffff); // Lui 1767 // 2 // Slli 1768 // 3 // Add 1769 int32_t low12 = ((Assembler::sextract(Assembler::ld_instr(insn_addr + MacroAssembler::instruction_size * 4), 31, 20))); // Addi/Jalr/Load. 1770 address ret = (address)(((intptr_t)upper18<<30ll) + ((intptr_t)mid18<<12ll) + low12); 1771 return ret; 1772 } 1773 1774 address MacroAssembler::get_target_of_li32(address insn_addr) { 1775 assert_cond(insn_addr != nullptr); 1776 intptr_t target_address = (((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12)) & 0xfffff) << 12; // Lui. 1777 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20)); // Addiw. 1778 return (address)target_address; 1779 } 1780 1781 // Patch any kind of instruction; there may be several instructions. 1782 // Return the total length (in bytes) of the instructions. 1783 int MacroAssembler::pd_patch_instruction_size(address instruction_address, address target) { 1784 assert_cond(instruction_address != nullptr); 1785 int64_t offset = target - instruction_address; 1786 if (MacroAssembler::is_jal_at(instruction_address)) { // jal 1787 return patch_offset_in_jal(instruction_address, offset); 1788 } else if (MacroAssembler::is_branch_at(instruction_address)) { // beq/bge/bgeu/blt/bltu/bne 1789 return patch_offset_in_conditional_branch(instruction_address, offset); 1790 } else if (MacroAssembler::is_pc_relative_at(instruction_address)) { // auipc, addi/jalr/load 1791 return patch_offset_in_pc_relative(instruction_address, offset); 1792 } else if (MacroAssembler::is_movptr1_at(instruction_address)) { // movptr1 1793 return patch_addr_in_movptr1(instruction_address, target); 1794 } else if (MacroAssembler::is_movptr2_at(instruction_address)) { // movptr2 1795 return patch_addr_in_movptr2(instruction_address, target); 1796 } else if (MacroAssembler::is_li32_at(instruction_address)) { // li32 1797 int64_t imm = (intptr_t)target; 1798 return patch_imm_in_li32(instruction_address, (int32_t)imm); 1799 } else if (MacroAssembler::is_li16u_at(instruction_address)) { 1800 int64_t imm = (intptr_t)target; 1801 return patch_imm_in_li16u(instruction_address, (uint16_t)imm); 1802 } else { 1803 #ifdef ASSERT 1804 tty->print_cr("pd_patch_instruction_size: instruction 0x%x at " INTPTR_FORMAT " could not be patched!\n", 1805 Assembler::ld_instr(instruction_address), p2i(instruction_address)); 1806 Disassembler::decode(instruction_address - 16, instruction_address + 16); 1807 #endif 1808 ShouldNotReachHere(); 1809 return -1; 1810 } 1811 } 1812 1813 address MacroAssembler::target_addr_for_insn(address insn_addr) { 1814 long offset = 0; 1815 assert_cond(insn_addr != nullptr); 1816 if (MacroAssembler::is_jal_at(insn_addr)) { // jal 1817 offset = get_offset_of_jal(insn_addr); 1818 } else if (MacroAssembler::is_branch_at(insn_addr)) { // beq/bge/bgeu/blt/bltu/bne 1819 offset = get_offset_of_conditional_branch(insn_addr); 1820 } else if (MacroAssembler::is_pc_relative_at(insn_addr)) { // auipc, addi/jalr/load 1821 offset = get_offset_of_pc_relative(insn_addr); 1822 } else if (MacroAssembler::is_movptr1_at(insn_addr)) { // movptr1 1823 return get_target_of_movptr1(insn_addr); 1824 } else if (MacroAssembler::is_movptr2_at(insn_addr)) { // movptr2 1825 return get_target_of_movptr2(insn_addr); 1826 } else if (MacroAssembler::is_li32_at(insn_addr)) { // li32 1827 return get_target_of_li32(insn_addr); 1828 } else { 1829 ShouldNotReachHere(); 1830 } 1831 return address(((uintptr_t)insn_addr + offset)); 1832 } 1833 1834 int MacroAssembler::patch_oop(address insn_addr, address o) { 1835 // OOPs are either narrow (32 bits) or wide (48 bits). We encode 1836 // narrow OOPs by setting the upper 16 bits in the first 1837 // instruction. 1838 if (MacroAssembler::is_li32_at(insn_addr)) { 1839 // Move narrow OOP 1840 uint32_t n = CompressedOops::narrow_oop_value(cast_to_oop(o)); 1841 return patch_imm_in_li32(insn_addr, (int32_t)n); 1842 } else if (MacroAssembler::is_movptr1_at(insn_addr)) { 1843 // Move wide OOP 1844 return patch_addr_in_movptr1(insn_addr, o); 1845 } else if (MacroAssembler::is_movptr2_at(insn_addr)) { 1846 // Move wide OOP 1847 return patch_addr_in_movptr2(insn_addr, o); 1848 } 1849 ShouldNotReachHere(); 1850 return -1; 1851 } 1852 1853 void MacroAssembler::reinit_heapbase() { 1854 if (UseCompressedOops) { 1855 if (Universe::is_fully_initialized()) { 1856 mv(xheapbase, CompressedOops::ptrs_base()); 1857 } else { 1858 ExternalAddress target(CompressedOops::ptrs_base_addr()); 1859 relocate(target.rspec(), [&] { 1860 int32_t offset; 1861 la(xheapbase, target.target(), offset); 1862 ld(xheapbase, Address(xheapbase, offset)); 1863 }); 1864 } 1865 } 1866 } 1867 1868 void MacroAssembler::movptr(Register Rd, address addr, Register temp) { 1869 int offset = 0; 1870 movptr(Rd, addr, offset, temp); 1871 addi(Rd, Rd, offset); 1872 } 1873 1874 void MacroAssembler::movptr(Register Rd, address addr, int32_t &offset, Register temp) { 1875 uint64_t uimm64 = (uint64_t)addr; 1876 #ifndef PRODUCT 1877 { 1878 char buffer[64]; 1879 snprintf(buffer, sizeof(buffer), "0x%" PRIx64, uimm64); 1880 block_comment(buffer); 1881 } 1882 #endif 1883 assert(uimm64 < (1ull << 48), "48-bit overflow in address constant"); 1884 1885 if (temp == noreg) { 1886 movptr1(Rd, uimm64, offset); 1887 } else { 1888 movptr2(Rd, uimm64, offset, temp); 1889 } 1890 } 1891 1892 void MacroAssembler::movptr1(Register Rd, uint64_t imm64, int32_t &offset) { 1893 // Load upper 31 bits 1894 // 1895 // In case of 11th bit of `lower` is 0, it's straightforward to understand. 1896 // In case of 11th bit of `lower` is 1, it's a bit tricky, to help understand, 1897 // imagine divide both `upper` and `lower` into 2 parts respectively, i.e. 1898 // [upper_20, upper_12], [lower_20, lower_12], they are the same just before 1899 // `lower = (lower << 52) >> 52;`. 1900 // After `upper -= lower;`, 1901 // upper_20' = upper_20 - (-1) == upper_20 + 1 1902 // upper_12 = 0x000 1903 // After `lui(Rd, upper);`, `Rd` = upper_20' << 12 1904 // Also divide `Rd` into 2 parts [Rd_20, Rd_12], 1905 // Rd_20 == upper_20' 1906 // Rd_12 == 0x000 1907 // After `addi(Rd, Rd, lower);`, 1908 // Rd_20 = upper_20' + (-1) == upper_20 + 1 - 1 = upper_20 1909 // Rd_12 = lower_12 1910 // So, finally Rd == [upper_20, lower_12] 1911 int64_t imm = imm64 >> 17; 1912 int64_t upper = imm, lower = imm; 1913 lower = (lower << 52) >> 52; 1914 upper -= lower; 1915 upper = (int32_t)upper; 1916 lui(Rd, upper); 1917 addi(Rd, Rd, lower); 1918 1919 // Load the rest 17 bits. 1920 slli(Rd, Rd, 11); 1921 addi(Rd, Rd, (imm64 >> 6) & 0x7ff); 1922 slli(Rd, Rd, 6); 1923 1924 // This offset will be used by following jalr/ld. 1925 offset = imm64 & 0x3f; 1926 } 1927 1928 void MacroAssembler::movptr2(Register Rd, uint64_t addr, int32_t &offset, Register tmp) { 1929 assert_different_registers(Rd, tmp, noreg); 1930 1931 // addr: [upper18, lower30[mid18, lower12]] 1932 1933 int64_t upper18 = addr >> 18; 1934 lui(tmp, upper18); 1935 1936 int64_t lower30 = addr & 0x3fffffff; 1937 int64_t mid18 = lower30, lower12 = lower30; 1938 lower12 = (lower12 << 52) >> 52; 1939 // For this tricky part (`mid18 -= lower12;` + `offset = lower12;`), 1940 // please refer to movptr1 above. 1941 mid18 -= (int32_t)lower12; 1942 lui(Rd, mid18); 1943 1944 slli(tmp, tmp, 18); 1945 add(Rd, Rd, tmp); 1946 1947 offset = lower12; 1948 } 1949 1950 void MacroAssembler::add(Register Rd, Register Rn, int64_t increment, Register temp) { 1951 if (is_simm12(increment)) { 1952 addi(Rd, Rn, increment); 1953 } else { 1954 assert_different_registers(Rn, temp); 1955 li(temp, increment); 1956 add(Rd, Rn, temp); 1957 } 1958 } 1959 1960 void MacroAssembler::addw(Register Rd, Register Rn, int32_t increment, Register temp) { 1961 if (is_simm12(increment)) { 1962 addiw(Rd, Rn, increment); 1963 } else { 1964 assert_different_registers(Rn, temp); 1965 li(temp, increment); 1966 addw(Rd, Rn, temp); 1967 } 1968 } 1969 1970 void MacroAssembler::sub(Register Rd, Register Rn, int64_t decrement, Register temp) { 1971 if (is_simm12(-decrement)) { 1972 addi(Rd, Rn, -decrement); 1973 } else { 1974 assert_different_registers(Rn, temp); 1975 li(temp, decrement); 1976 sub(Rd, Rn, temp); 1977 } 1978 } 1979 1980 void MacroAssembler::subw(Register Rd, Register Rn, int32_t decrement, Register temp) { 1981 if (is_simm12(-decrement)) { 1982 addiw(Rd, Rn, -decrement); 1983 } else { 1984 assert_different_registers(Rn, temp); 1985 li(temp, decrement); 1986 subw(Rd, Rn, temp); 1987 } 1988 } 1989 1990 void MacroAssembler::andrw(Register Rd, Register Rs1, Register Rs2) { 1991 andr(Rd, Rs1, Rs2); 1992 sign_extend(Rd, Rd, 32); 1993 } 1994 1995 void MacroAssembler::orrw(Register Rd, Register Rs1, Register Rs2) { 1996 orr(Rd, Rs1, Rs2); 1997 sign_extend(Rd, Rd, 32); 1998 } 1999 2000 void MacroAssembler::xorrw(Register Rd, Register Rs1, Register Rs2) { 2001 xorr(Rd, Rs1, Rs2); 2002 sign_extend(Rd, Rd, 32); 2003 } 2004 2005 // Rd = Rs1 & (~Rd2) 2006 void MacroAssembler::andn(Register Rd, Register Rs1, Register Rs2) { 2007 if (UseZbb) { 2008 Assembler::andn(Rd, Rs1, Rs2); 2009 return; 2010 } 2011 2012 notr(Rd, Rs2); 2013 andr(Rd, Rs1, Rd); 2014 } 2015 2016 // Rd = Rs1 | (~Rd2) 2017 void MacroAssembler::orn(Register Rd, Register Rs1, Register Rs2) { 2018 if (UseZbb) { 2019 Assembler::orn(Rd, Rs1, Rs2); 2020 return; 2021 } 2022 2023 notr(Rd, Rs2); 2024 orr(Rd, Rs1, Rd); 2025 } 2026 2027 // Note: load_unsigned_short used to be called load_unsigned_word. 2028 int MacroAssembler::load_unsigned_short(Register dst, Address src) { 2029 int off = offset(); 2030 lhu(dst, src); 2031 return off; 2032 } 2033 2034 int MacroAssembler::load_unsigned_byte(Register dst, Address src) { 2035 int off = offset(); 2036 lbu(dst, src); 2037 return off; 2038 } 2039 2040 int MacroAssembler::load_signed_short(Register dst, Address src) { 2041 int off = offset(); 2042 lh(dst, src); 2043 return off; 2044 } 2045 2046 int MacroAssembler::load_signed_byte(Register dst, Address src) { 2047 int off = offset(); 2048 lb(dst, src); 2049 return off; 2050 } 2051 2052 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed) { 2053 switch (size_in_bytes) { 2054 case 8: ld(dst, src); break; 2055 case 4: is_signed ? lw(dst, src) : lwu(dst, src); break; 2056 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; 2057 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; 2058 default: ShouldNotReachHere(); 2059 } 2060 } 2061 2062 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes) { 2063 switch (size_in_bytes) { 2064 case 8: sd(src, dst); break; 2065 case 4: sw(src, dst); break; 2066 case 2: sh(src, dst); break; 2067 case 1: sb(src, dst); break; 2068 default: ShouldNotReachHere(); 2069 } 2070 } 2071 2072 // granularity is 1 OR 2 bytes per load. dst and src.base() allowed to be the same register 2073 void MacroAssembler::load_short_misaligned(Register dst, Address src, Register tmp, bool is_signed, int granularity) { 2074 if (granularity != 1 && granularity != 2) { 2075 ShouldNotReachHere(); 2076 } 2077 if (AvoidUnalignedAccesses && (granularity != 2)) { 2078 assert_different_registers(dst, tmp); 2079 assert_different_registers(tmp, src.base()); 2080 is_signed ? lb(tmp, Address(src.base(), src.offset() + 1)) : lbu(tmp, Address(src.base(), src.offset() + 1)); 2081 slli(tmp, tmp, 8); 2082 lbu(dst, src); 2083 add(dst, dst, tmp); 2084 } else { 2085 is_signed ? lh(dst, src) : lhu(dst, src); 2086 } 2087 } 2088 2089 // granularity is 1, 2 OR 4 bytes per load, if granularity 2 or 4 then dst and src.base() allowed to be the same register 2090 void MacroAssembler::load_int_misaligned(Register dst, Address src, Register tmp, bool is_signed, int granularity) { 2091 if (AvoidUnalignedAccesses && (granularity != 4)) { 2092 switch(granularity) { 2093 case 1: 2094 assert_different_registers(dst, tmp, src.base()); 2095 lbu(dst, src); 2096 lbu(tmp, Address(src.base(), src.offset() + 1)); 2097 slli(tmp, tmp, 8); 2098 add(dst, dst, tmp); 2099 lbu(tmp, Address(src.base(), src.offset() + 2)); 2100 slli(tmp, tmp, 16); 2101 add(dst, dst, tmp); 2102 is_signed ? lb(tmp, Address(src.base(), src.offset() + 3)) : lbu(tmp, Address(src.base(), src.offset() + 3)); 2103 slli(tmp, tmp, 24); 2104 add(dst, dst, tmp); 2105 break; 2106 case 2: 2107 assert_different_registers(dst, tmp); 2108 assert_different_registers(tmp, src.base()); 2109 is_signed ? lh(tmp, Address(src.base(), src.offset() + 2)) : lhu(tmp, Address(src.base(), src.offset() + 2)); 2110 slli(tmp, tmp, 16); 2111 lhu(dst, src); 2112 add(dst, dst, tmp); 2113 break; 2114 default: 2115 ShouldNotReachHere(); 2116 } 2117 } else { 2118 is_signed ? lw(dst, src) : lwu(dst, src); 2119 } 2120 } 2121 2122 // granularity is 1, 2, 4 or 8 bytes per load, if granularity 4 or 8 then dst and src.base() allowed to be same register 2123 void MacroAssembler::load_long_misaligned(Register dst, Address src, Register tmp, int granularity) { 2124 if (AvoidUnalignedAccesses && (granularity != 8)) { 2125 switch(granularity){ 2126 case 1: 2127 assert_different_registers(dst, tmp, src.base()); 2128 lbu(dst, src); 2129 lbu(tmp, Address(src.base(), src.offset() + 1)); 2130 slli(tmp, tmp, 8); 2131 add(dst, dst, tmp); 2132 lbu(tmp, Address(src.base(), src.offset() + 2)); 2133 slli(tmp, tmp, 16); 2134 add(dst, dst, tmp); 2135 lbu(tmp, Address(src.base(), src.offset() + 3)); 2136 slli(tmp, tmp, 24); 2137 add(dst, dst, tmp); 2138 lbu(tmp, Address(src.base(), src.offset() + 4)); 2139 slli(tmp, tmp, 32); 2140 add(dst, dst, tmp); 2141 lbu(tmp, Address(src.base(), src.offset() + 5)); 2142 slli(tmp, tmp, 40); 2143 add(dst, dst, tmp); 2144 lbu(tmp, Address(src.base(), src.offset() + 6)); 2145 slli(tmp, tmp, 48); 2146 add(dst, dst, tmp); 2147 lbu(tmp, Address(src.base(), src.offset() + 7)); 2148 slli(tmp, tmp, 56); 2149 add(dst, dst, tmp); 2150 break; 2151 case 2: 2152 assert_different_registers(dst, tmp, src.base()); 2153 lhu(dst, src); 2154 lhu(tmp, Address(src.base(), src.offset() + 2)); 2155 slli(tmp, tmp, 16); 2156 add(dst, dst, tmp); 2157 lhu(tmp, Address(src.base(), src.offset() + 4)); 2158 slli(tmp, tmp, 32); 2159 add(dst, dst, tmp); 2160 lhu(tmp, Address(src.base(), src.offset() + 6)); 2161 slli(tmp, tmp, 48); 2162 add(dst, dst, tmp); 2163 break; 2164 case 4: 2165 assert_different_registers(dst, tmp); 2166 assert_different_registers(tmp, src.base()); 2167 lwu(tmp, Address(src.base(), src.offset() + 4)); 2168 slli(tmp, tmp, 32); 2169 lwu(dst, src); 2170 add(dst, dst, tmp); 2171 break; 2172 default: 2173 ShouldNotReachHere(); 2174 } 2175 } else { 2176 ld(dst, src); 2177 } 2178 } 2179 2180 2181 // reverse bytes in halfword in lower 16 bits and sign-extend 2182 // Rd[15:0] = Rs[7:0] Rs[15:8] (sign-extend to 64 bits) 2183 void MacroAssembler::revb_h_h(Register Rd, Register Rs, Register tmp) { 2184 if (UseZbb) { 2185 rev8(Rd, Rs); 2186 srai(Rd, Rd, 48); 2187 return; 2188 } 2189 assert_different_registers(Rs, tmp); 2190 assert_different_registers(Rd, tmp); 2191 srli(tmp, Rs, 8); 2192 andi(tmp, tmp, 0xFF); 2193 slli(Rd, Rs, 56); 2194 srai(Rd, Rd, 48); // sign-extend 2195 orr(Rd, Rd, tmp); 2196 } 2197 2198 // reverse bytes in lower word and sign-extend 2199 // Rd[31:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] (sign-extend to 64 bits) 2200 void MacroAssembler::revb_w_w(Register Rd, Register Rs, Register tmp1, Register tmp2) { 2201 if (UseZbb) { 2202 rev8(Rd, Rs); 2203 srai(Rd, Rd, 32); 2204 return; 2205 } 2206 assert_different_registers(Rs, tmp1, tmp2); 2207 assert_different_registers(Rd, tmp1, tmp2); 2208 revb_h_w_u(Rd, Rs, tmp1, tmp2); 2209 slli(tmp2, Rd, 48); 2210 srai(tmp2, tmp2, 32); // sign-extend 2211 srli(Rd, Rd, 16); 2212 orr(Rd, Rd, tmp2); 2213 } 2214 2215 // reverse bytes in halfword in lower 16 bits and zero-extend 2216 // Rd[15:0] = Rs[7:0] Rs[15:8] (zero-extend to 64 bits) 2217 void MacroAssembler::revb_h_h_u(Register Rd, Register Rs, Register tmp) { 2218 if (UseZbb) { 2219 rev8(Rd, Rs); 2220 srli(Rd, Rd, 48); 2221 return; 2222 } 2223 assert_different_registers(Rs, tmp); 2224 assert_different_registers(Rd, tmp); 2225 srli(tmp, Rs, 8); 2226 andi(tmp, tmp, 0xFF); 2227 andi(Rd, Rs, 0xFF); 2228 slli(Rd, Rd, 8); 2229 orr(Rd, Rd, tmp); 2230 } 2231 2232 // reverse bytes in halfwords in lower 32 bits and zero-extend 2233 // Rd[31:0] = Rs[23:16] Rs[31:24] Rs[7:0] Rs[15:8] (zero-extend to 64 bits) 2234 void MacroAssembler::revb_h_w_u(Register Rd, Register Rs, Register tmp1, Register tmp2) { 2235 if (UseZbb) { 2236 rev8(Rd, Rs); 2237 rori(Rd, Rd, 32); 2238 roriw(Rd, Rd, 16); 2239 zero_extend(Rd, Rd, 32); 2240 return; 2241 } 2242 assert_different_registers(Rs, tmp1, tmp2); 2243 assert_different_registers(Rd, tmp1, tmp2); 2244 srli(tmp2, Rs, 16); 2245 revb_h_h_u(tmp2, tmp2, tmp1); 2246 revb_h_h_u(Rd, Rs, tmp1); 2247 slli(tmp2, tmp2, 16); 2248 orr(Rd, Rd, tmp2); 2249 } 2250 2251 // This method is only used for revb_h 2252 // Rd = Rs[47:0] Rs[55:48] Rs[63:56] 2253 void MacroAssembler::revb_h_helper(Register Rd, Register Rs, Register tmp1, Register tmp2) { 2254 assert_different_registers(Rs, tmp1, tmp2); 2255 assert_different_registers(Rd, tmp1); 2256 srli(tmp1, Rs, 48); 2257 andi(tmp2, tmp1, 0xFF); 2258 slli(tmp2, tmp2, 8); 2259 srli(tmp1, tmp1, 8); 2260 orr(tmp1, tmp1, tmp2); 2261 slli(Rd, Rs, 16); 2262 orr(Rd, Rd, tmp1); 2263 } 2264 2265 // reverse bytes in each halfword 2266 // Rd[63:0] = Rs[55:48] Rs[63:56] Rs[39:32] Rs[47:40] Rs[23:16] Rs[31:24] Rs[7:0] Rs[15:8] 2267 void MacroAssembler::revb_h(Register Rd, Register Rs, Register tmp1, Register tmp2) { 2268 if (UseZbb) { 2269 assert_different_registers(Rs, tmp1); 2270 assert_different_registers(Rd, tmp1); 2271 rev8(Rd, Rs); 2272 zero_extend(tmp1, Rd, 32); 2273 roriw(tmp1, tmp1, 16); 2274 slli(tmp1, tmp1, 32); 2275 srli(Rd, Rd, 32); 2276 roriw(Rd, Rd, 16); 2277 zero_extend(Rd, Rd, 32); 2278 orr(Rd, Rd, tmp1); 2279 return; 2280 } 2281 assert_different_registers(Rs, tmp1, tmp2); 2282 assert_different_registers(Rd, tmp1, tmp2); 2283 revb_h_helper(Rd, Rs, tmp1, tmp2); 2284 for (int i = 0; i < 3; ++i) { 2285 revb_h_helper(Rd, Rd, tmp1, tmp2); 2286 } 2287 } 2288 2289 // reverse bytes in each word 2290 // Rd[63:0] = Rs[39:32] Rs[47:40] Rs[55:48] Rs[63:56] Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] 2291 void MacroAssembler::revb_w(Register Rd, Register Rs, Register tmp1, Register tmp2) { 2292 if (UseZbb) { 2293 rev8(Rd, Rs); 2294 rori(Rd, Rd, 32); 2295 return; 2296 } 2297 assert_different_registers(Rs, tmp1, tmp2); 2298 assert_different_registers(Rd, tmp1, tmp2); 2299 revb(Rd, Rs, tmp1, tmp2); 2300 ror_imm(Rd, Rd, 32); 2301 } 2302 2303 // reverse bytes in doubleword 2304 // Rd[63:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] Rs[39:32] Rs[47,40] Rs[55,48] Rs[63:56] 2305 void MacroAssembler::revb(Register Rd, Register Rs, Register tmp1, Register tmp2) { 2306 if (UseZbb) { 2307 rev8(Rd, Rs); 2308 return; 2309 } 2310 assert_different_registers(Rs, tmp1, tmp2); 2311 assert_different_registers(Rd, tmp1, tmp2); 2312 andi(tmp1, Rs, 0xFF); 2313 slli(tmp1, tmp1, 8); 2314 for (int step = 8; step < 56; step += 8) { 2315 srli(tmp2, Rs, step); 2316 andi(tmp2, tmp2, 0xFF); 2317 orr(tmp1, tmp1, tmp2); 2318 slli(tmp1, tmp1, 8); 2319 } 2320 srli(Rd, Rs, 56); 2321 andi(Rd, Rd, 0xFF); 2322 orr(Rd, tmp1, Rd); 2323 } 2324 2325 // rotate right with shift bits 2326 void MacroAssembler::ror_imm(Register dst, Register src, uint32_t shift, Register tmp) 2327 { 2328 if (UseZbb) { 2329 rori(dst, src, shift); 2330 return; 2331 } 2332 2333 assert_different_registers(dst, tmp); 2334 assert_different_registers(src, tmp); 2335 assert(shift < 64, "shift amount must be < 64"); 2336 slli(tmp, src, 64 - shift); 2337 srli(dst, src, shift); 2338 orr(dst, dst, tmp); 2339 } 2340 2341 // rotate left with shift bits, 32-bit version 2342 void MacroAssembler::rolw_imm(Register dst, Register src, uint32_t shift, Register tmp) { 2343 if (UseZbb) { 2344 // no roliw available 2345 roriw(dst, src, 32 - shift); 2346 return; 2347 } 2348 2349 assert_different_registers(dst, tmp); 2350 assert_different_registers(src, tmp); 2351 assert(shift < 32, "shift amount must be < 32"); 2352 srliw(tmp, src, 32 - shift); 2353 slliw(dst, src, shift); 2354 orr(dst, dst, tmp); 2355 } 2356 2357 void MacroAssembler::andi(Register Rd, Register Rn, int64_t imm, Register tmp) { 2358 if (is_simm12(imm)) { 2359 and_imm12(Rd, Rn, imm); 2360 } else { 2361 assert_different_registers(Rn, tmp); 2362 mv(tmp, imm); 2363 andr(Rd, Rn, tmp); 2364 } 2365 } 2366 2367 void MacroAssembler::orptr(Address adr, RegisterOrConstant src, Register tmp1, Register tmp2) { 2368 ld(tmp1, adr); 2369 if (src.is_register()) { 2370 orr(tmp1, tmp1, src.as_register()); 2371 } else { 2372 if (is_simm12(src.as_constant())) { 2373 ori(tmp1, tmp1, src.as_constant()); 2374 } else { 2375 assert_different_registers(tmp1, tmp2); 2376 mv(tmp2, src.as_constant()); 2377 orr(tmp1, tmp1, tmp2); 2378 } 2379 } 2380 sd(tmp1, adr); 2381 } 2382 2383 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp1, Register tmp2, Label &L) { 2384 assert_different_registers(oop, trial_klass, tmp1, tmp2); 2385 if (UseCompressedClassPointers) { 2386 lwu(tmp1, Address(oop, oopDesc::klass_offset_in_bytes())); 2387 if (CompressedKlassPointers::base() == nullptr) { 2388 slli(tmp1, tmp1, CompressedKlassPointers::shift()); 2389 beq(trial_klass, tmp1, L); 2390 return; 2391 } 2392 decode_klass_not_null(tmp1, tmp2); 2393 } else { 2394 ld(tmp1, Address(oop, oopDesc::klass_offset_in_bytes())); 2395 } 2396 beq(trial_klass, tmp1, L); 2397 } 2398 2399 // Move an oop into a register. 2400 void MacroAssembler::movoop(Register dst, jobject obj) { 2401 int oop_index; 2402 if (obj == nullptr) { 2403 oop_index = oop_recorder()->allocate_oop_index(obj); 2404 } else { 2405 #ifdef ASSERT 2406 { 2407 ThreadInVMfromUnknown tiv; 2408 assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop"); 2409 } 2410 #endif 2411 oop_index = oop_recorder()->find_index(obj); 2412 } 2413 RelocationHolder rspec = oop_Relocation::spec(oop_index); 2414 2415 if (BarrierSet::barrier_set()->barrier_set_assembler()->supports_instruction_patching()) { 2416 la(dst, Address((address)obj, rspec)); 2417 } else { 2418 address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address 2419 ld_constant(dst, Address(dummy, rspec)); 2420 } 2421 } 2422 2423 // Move a metadata address into a register. 2424 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 2425 assert((uintptr_t)obj < (1ull << 48), "48-bit overflow in metadata"); 2426 int oop_index; 2427 if (obj == nullptr) { 2428 oop_index = oop_recorder()->allocate_metadata_index(obj); 2429 } else { 2430 oop_index = oop_recorder()->find_index(obj); 2431 } 2432 RelocationHolder rspec = metadata_Relocation::spec(oop_index); 2433 la(dst, Address((address)obj, rspec)); 2434 } 2435 2436 // Writes to stack successive pages until offset reached to check for 2437 // stack overflow + shadow pages. This clobbers tmp. 2438 void MacroAssembler::bang_stack_size(Register size, Register tmp) { 2439 assert_different_registers(tmp, size, t0); 2440 // Bang stack for total size given plus shadow page size. 2441 // Bang one page at a time because large size can bang beyond yellow and 2442 // red zones. 2443 mv(t0, (int)os::vm_page_size()); 2444 Label loop; 2445 bind(loop); 2446 sub(tmp, sp, t0); 2447 subw(size, size, t0); 2448 sd(size, Address(tmp)); 2449 bgtz(size, loop); 2450 2451 // Bang down shadow pages too. 2452 // At this point, (tmp-0) is the last address touched, so don't 2453 // touch it again. (It was touched as (tmp-pagesize) but then tmp 2454 // was post-decremented.) Skip this address by starting at i=1, and 2455 // touch a few more pages below. N.B. It is important to touch all 2456 // the way down to and including i=StackShadowPages. 2457 for (int i = 0; i < (int)(StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()) - 1; i++) { 2458 // this could be any sized move but this is can be a debugging crumb 2459 // so the bigger the better. 2460 sub(tmp, tmp, (int)os::vm_page_size()); 2461 sd(size, Address(tmp, 0)); 2462 } 2463 } 2464 2465 SkipIfEqual::SkipIfEqual(MacroAssembler* masm, const bool* flag_addr, bool value) { 2466 int32_t offset = 0; 2467 _masm = masm; 2468 ExternalAddress target((address)flag_addr); 2469 _masm->relocate(target.rspec(), [&] { 2470 int32_t offset; 2471 _masm->la(t0, target.target(), offset); 2472 _masm->lbu(t0, Address(t0, offset)); 2473 }); 2474 if (value) { 2475 _masm->bnez(t0, _label); 2476 } else { 2477 _masm->beqz(t0, _label); 2478 } 2479 } 2480 2481 SkipIfEqual::~SkipIfEqual() { 2482 _masm->bind(_label); 2483 _masm = nullptr; 2484 } 2485 2486 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, Register tmp2) { 2487 const int mirror_offset = in_bytes(Klass::java_mirror_offset()); 2488 ld(dst, Address(xmethod, Method::const_offset())); 2489 ld(dst, Address(dst, ConstMethod::constants_offset())); 2490 ld(dst, Address(dst, ConstantPool::pool_holder_offset())); 2491 ld(dst, Address(dst, mirror_offset)); 2492 resolve_oop_handle(dst, tmp1, tmp2); 2493 } 2494 2495 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) { 2496 // OopHandle::resolve is an indirection. 2497 assert_different_registers(result, tmp1, tmp2); 2498 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp1, tmp2); 2499 } 2500 2501 // ((WeakHandle)result).resolve() 2502 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2) { 2503 assert_different_registers(result, tmp1, tmp2); 2504 Label resolved; 2505 2506 // A null weak handle resolves to null. 2507 beqz(result, resolved); 2508 2509 // Only 64 bit platforms support GCs that require a tmp register 2510 // Only IN_HEAP loads require a thread_tmp register 2511 // WeakHandle::resolve is an indirection like jweak. 2512 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 2513 result, Address(result), tmp1, tmp2); 2514 bind(resolved); 2515 } 2516 2517 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, 2518 Register dst, Address src, 2519 Register tmp1, Register tmp2) { 2520 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 2521 decorators = AccessInternal::decorator_fixup(decorators, type); 2522 bool as_raw = (decorators & AS_RAW) != 0; 2523 if (as_raw) { 2524 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2); 2525 } else { 2526 bs->load_at(this, decorators, type, dst, src, tmp1, tmp2); 2527 } 2528 } 2529 2530 void MacroAssembler::null_check(Register reg, int offset) { 2531 if (needs_explicit_null_check(offset)) { 2532 // provoke OS null exception if reg is null by 2533 // accessing M[reg] w/o changing any registers 2534 // NOTE: this is plenty to provoke a segv 2535 ld(zr, Address(reg, 0)); 2536 } else { 2537 // nothing to do, (later) access of M[reg + offset] 2538 // will provoke OS null exception if reg is null 2539 } 2540 } 2541 2542 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, 2543 Address dst, Register val, 2544 Register tmp1, Register tmp2, Register tmp3) { 2545 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 2546 decorators = AccessInternal::decorator_fixup(decorators, type); 2547 bool as_raw = (decorators & AS_RAW) != 0; 2548 if (as_raw) { 2549 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 2550 } else { 2551 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 2552 } 2553 } 2554 2555 // Algorithm must match CompressedOops::encode. 2556 void MacroAssembler::encode_heap_oop(Register d, Register s) { 2557 verify_oop_msg(s, "broken oop in encode_heap_oop"); 2558 if (CompressedOops::base() == nullptr) { 2559 if (CompressedOops::shift() != 0) { 2560 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 2561 srli(d, s, LogMinObjAlignmentInBytes); 2562 } else { 2563 mv(d, s); 2564 } 2565 } else { 2566 Label notNull; 2567 sub(d, s, xheapbase); 2568 bgez(d, notNull); 2569 mv(d, zr); 2570 bind(notNull); 2571 if (CompressedOops::shift() != 0) { 2572 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 2573 srli(d, d, CompressedOops::shift()); 2574 } 2575 } 2576 } 2577 2578 void MacroAssembler::encode_heap_oop_not_null(Register r) { 2579 #ifdef ASSERT 2580 if (CheckCompressedOops) { 2581 Label ok; 2582 bnez(r, ok); 2583 stop("null oop passed to encode_heap_oop_not_null"); 2584 bind(ok); 2585 } 2586 #endif 2587 verify_oop_msg(r, "broken oop in encode_heap_oop_not_null"); 2588 if (CompressedOops::base() != nullptr) { 2589 sub(r, r, xheapbase); 2590 } 2591 if (CompressedOops::shift() != 0) { 2592 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 2593 srli(r, r, LogMinObjAlignmentInBytes); 2594 } 2595 } 2596 2597 void MacroAssembler::encode_heap_oop_not_null(Register dst, Register src) { 2598 #ifdef ASSERT 2599 if (CheckCompressedOops) { 2600 Label ok; 2601 bnez(src, ok); 2602 stop("null oop passed to encode_heap_oop_not_null2"); 2603 bind(ok); 2604 } 2605 #endif 2606 verify_oop_msg(src, "broken oop in encode_heap_oop_not_null2"); 2607 2608 Register data = src; 2609 if (CompressedOops::base() != nullptr) { 2610 sub(dst, src, xheapbase); 2611 data = dst; 2612 } 2613 if (CompressedOops::shift() != 0) { 2614 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 2615 srli(dst, data, LogMinObjAlignmentInBytes); 2616 data = dst; 2617 } 2618 if (data == src) { 2619 mv(dst, src); 2620 } 2621 } 2622 2623 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { 2624 assert_different_registers(dst, tmp); 2625 assert_different_registers(src, tmp); 2626 if (UseCompressedClassPointers) { 2627 lwu(dst, Address(src, oopDesc::klass_offset_in_bytes())); 2628 decode_klass_not_null(dst, tmp); 2629 } else { 2630 ld(dst, Address(src, oopDesc::klass_offset_in_bytes())); 2631 } 2632 } 2633 2634 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { 2635 // FIXME: Should this be a store release? concurrent gcs assumes 2636 // klass length is valid if klass field is not null. 2637 if (UseCompressedClassPointers) { 2638 encode_klass_not_null(src, tmp); 2639 sw(src, Address(dst, oopDesc::klass_offset_in_bytes())); 2640 } else { 2641 sd(src, Address(dst, oopDesc::klass_offset_in_bytes())); 2642 } 2643 } 2644 2645 void MacroAssembler::store_klass_gap(Register dst, Register src) { 2646 if (UseCompressedClassPointers) { 2647 // Store to klass gap in destination 2648 sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes())); 2649 } 2650 } 2651 2652 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) { 2653 assert_different_registers(r, tmp); 2654 decode_klass_not_null(r, r, tmp); 2655 } 2656 2657 void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register tmp) { 2658 assert(UseCompressedClassPointers, "should only be used for compressed headers"); 2659 2660 if (CompressedKlassPointers::base() == nullptr) { 2661 if (CompressedKlassPointers::shift() != 0) { 2662 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 2663 slli(dst, src, LogKlassAlignmentInBytes); 2664 } else { 2665 mv(dst, src); 2666 } 2667 return; 2668 } 2669 2670 Register xbase = dst; 2671 if (dst == src) { 2672 xbase = tmp; 2673 } 2674 2675 assert_different_registers(src, xbase); 2676 mv(xbase, (uintptr_t)CompressedKlassPointers::base()); 2677 2678 if (CompressedKlassPointers::shift() != 0) { 2679 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 2680 assert_different_registers(t0, xbase); 2681 shadd(dst, src, xbase, t0, LogKlassAlignmentInBytes); 2682 } else { 2683 add(dst, xbase, src); 2684 } 2685 } 2686 2687 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) { 2688 assert_different_registers(r, tmp); 2689 encode_klass_not_null(r, r, tmp); 2690 } 2691 2692 void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register tmp) { 2693 assert(UseCompressedClassPointers, "should only be used for compressed headers"); 2694 2695 if (CompressedKlassPointers::base() == nullptr) { 2696 if (CompressedKlassPointers::shift() != 0) { 2697 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 2698 srli(dst, src, LogKlassAlignmentInBytes); 2699 } else { 2700 mv(dst, src); 2701 } 2702 return; 2703 } 2704 2705 if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0 && 2706 CompressedKlassPointers::shift() == 0) { 2707 zero_extend(dst, src, 32); 2708 return; 2709 } 2710 2711 Register xbase = dst; 2712 if (dst == src) { 2713 xbase = tmp; 2714 } 2715 2716 assert_different_registers(src, xbase); 2717 mv(xbase, (uintptr_t)CompressedKlassPointers::base()); 2718 sub(dst, src, xbase); 2719 if (CompressedKlassPointers::shift() != 0) { 2720 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 2721 srli(dst, dst, LogKlassAlignmentInBytes); 2722 } 2723 } 2724 2725 void MacroAssembler::decode_heap_oop_not_null(Register r) { 2726 decode_heap_oop_not_null(r, r); 2727 } 2728 2729 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { 2730 assert(UseCompressedOops, "should only be used for compressed headers"); 2731 assert(Universe::heap() != nullptr, "java heap should be initialized"); 2732 // Cannot assert, unverified entry point counts instructions (see .ad file) 2733 // vtableStubs also counts instructions in pd_code_size_limit. 2734 // Also do not verify_oop as this is called by verify_oop. 2735 if (CompressedOops::shift() != 0) { 2736 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 2737 slli(dst, src, LogMinObjAlignmentInBytes); 2738 if (CompressedOops::base() != nullptr) { 2739 add(dst, xheapbase, dst); 2740 } 2741 } else { 2742 assert(CompressedOops::base() == nullptr, "sanity"); 2743 mv(dst, src); 2744 } 2745 } 2746 2747 void MacroAssembler::decode_heap_oop(Register d, Register s) { 2748 if (CompressedOops::base() == nullptr) { 2749 if (CompressedOops::shift() != 0 || d != s) { 2750 slli(d, s, CompressedOops::shift()); 2751 } 2752 } else { 2753 Label done; 2754 mv(d, s); 2755 beqz(s, done); 2756 shadd(d, s, xheapbase, d, LogMinObjAlignmentInBytes); 2757 bind(done); 2758 } 2759 verify_oop_msg(d, "broken oop in decode_heap_oop"); 2760 } 2761 2762 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1, 2763 Register tmp2, Register tmp3, DecoratorSet decorators) { 2764 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3); 2765 } 2766 2767 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, 2768 Register tmp2, DecoratorSet decorators) { 2769 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2); 2770 } 2771 2772 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, 2773 Register tmp2, DecoratorSet decorators) { 2774 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL, dst, src, tmp1, tmp2); 2775 } 2776 2777 // Used for storing nulls. 2778 void MacroAssembler::store_heap_oop_null(Address dst) { 2779 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg); 2780 } 2781 2782 int MacroAssembler::corrected_idivl(Register result, Register rs1, Register rs2, 2783 bool want_remainder, bool is_signed) 2784 { 2785 // Full implementation of Java idiv and irem. The function 2786 // returns the (pc) offset of the div instruction - may be needed 2787 // for implicit exceptions. 2788 // 2789 // input : rs1: dividend 2790 // rs2: divisor 2791 // 2792 // result: either 2793 // quotient (= rs1 idiv rs2) 2794 // remainder (= rs1 irem rs2) 2795 2796 2797 int idivl_offset = offset(); 2798 if (!want_remainder) { 2799 if (is_signed) { 2800 divw(result, rs1, rs2); 2801 } else { 2802 divuw(result, rs1, rs2); 2803 } 2804 } else { 2805 // result = rs1 % rs2; 2806 if (is_signed) { 2807 remw(result, rs1, rs2); 2808 } else { 2809 remuw(result, rs1, rs2); 2810 } 2811 } 2812 return idivl_offset; 2813 } 2814 2815 int MacroAssembler::corrected_idivq(Register result, Register rs1, Register rs2, 2816 bool want_remainder, bool is_signed) 2817 { 2818 // Full implementation of Java ldiv and lrem. The function 2819 // returns the (pc) offset of the div instruction - may be needed 2820 // for implicit exceptions. 2821 // 2822 // input : rs1: dividend 2823 // rs2: divisor 2824 // 2825 // result: either 2826 // quotient (= rs1 idiv rs2) 2827 // remainder (= rs1 irem rs2) 2828 2829 int idivq_offset = offset(); 2830 if (!want_remainder) { 2831 if (is_signed) { 2832 div(result, rs1, rs2); 2833 } else { 2834 divu(result, rs1, rs2); 2835 } 2836 } else { 2837 // result = rs1 % rs2; 2838 if (is_signed) { 2839 rem(result, rs1, rs2); 2840 } else { 2841 remu(result, rs1, rs2); 2842 } 2843 } 2844 return idivq_offset; 2845 } 2846 2847 // Look up the method for a megamorpic invkkeinterface call. 2848 // The target method is determined by <intf_klass, itable_index>. 2849 // The receiver klass is in recv_klass. 2850 // On success, the result will be in method_result, and execution falls through. 2851 // On failure, execution transfers to the given label. 2852 void MacroAssembler::lookup_interface_method(Register recv_klass, 2853 Register intf_klass, 2854 RegisterOrConstant itable_index, 2855 Register method_result, 2856 Register scan_tmp, 2857 Label& L_no_such_interface, 2858 bool return_method) { 2859 assert_different_registers(recv_klass, intf_klass, scan_tmp); 2860 assert_different_registers(method_result, intf_klass, scan_tmp); 2861 assert(recv_klass != method_result || !return_method, 2862 "recv_klass can be destroyed when mehtid isn't needed"); 2863 assert(itable_index.is_constant() || itable_index.as_register() == method_result, 2864 "caller must be same register for non-constant itable index as for method"); 2865 2866 // Compute start of first itableOffsetEntry (which is at the end of the vtable). 2867 int vtable_base = in_bytes(Klass::vtable_start_offset()); 2868 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 2869 int scan_step = itableOffsetEntry::size() * wordSize; 2870 int vte_size = vtableEntry::size_in_bytes(); 2871 assert(vte_size == wordSize, "else adjust times_vte_scale"); 2872 2873 lwu(scan_tmp, Address(recv_klass, Klass::vtable_length_offset())); 2874 2875 // Could store the aligned, prescaled offset in the klass. 2876 shadd(scan_tmp, scan_tmp, recv_klass, scan_tmp, 3); 2877 add(scan_tmp, scan_tmp, vtable_base); 2878 2879 if (return_method) { 2880 // Adjust recv_klass by scaled itable_index, so we can free itable_index. 2881 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 2882 if (itable_index.is_register()) { 2883 slli(t0, itable_index.as_register(), 3); 2884 } else { 2885 mv(t0, itable_index.as_constant() << 3); 2886 } 2887 add(recv_klass, recv_klass, t0); 2888 if (itentry_off) { 2889 add(recv_klass, recv_klass, itentry_off); 2890 } 2891 } 2892 2893 Label search, found_method; 2894 2895 ld(method_result, Address(scan_tmp, itableOffsetEntry::interface_offset())); 2896 beq(intf_klass, method_result, found_method); 2897 bind(search); 2898 // Check that the previous entry is non-null. A null entry means that 2899 // the receiver class doesn't implement the interface, and wasn't the 2900 // same as when the caller was compiled. 2901 beqz(method_result, L_no_such_interface, /* is_far */ true); 2902 addi(scan_tmp, scan_tmp, scan_step); 2903 ld(method_result, Address(scan_tmp, itableOffsetEntry::interface_offset())); 2904 bne(intf_klass, method_result, search); 2905 2906 bind(found_method); 2907 2908 // Got a hit. 2909 if (return_method) { 2910 lwu(scan_tmp, Address(scan_tmp, itableOffsetEntry::offset_offset())); 2911 add(method_result, recv_klass, scan_tmp); 2912 ld(method_result, Address(method_result)); 2913 } 2914 } 2915 2916 // Look up the method for a megamorphic invokeinterface call in a single pass over itable: 2917 // - check recv_klass (actual object class) is a subtype of resolved_klass from CompiledICData 2918 // - find a holder_klass (class that implements the method) vtable offset and get the method from vtable by index 2919 // The target method is determined by <holder_klass, itable_index>. 2920 // The receiver klass is in recv_klass. 2921 // On success, the result will be in method_result, and execution falls through. 2922 // On failure, execution transfers to the given label. 2923 void MacroAssembler::lookup_interface_method_stub(Register recv_klass, 2924 Register holder_klass, 2925 Register resolved_klass, 2926 Register method_result, 2927 Register temp_itbl_klass, 2928 Register scan_temp, 2929 int itable_index, 2930 Label& L_no_such_interface) { 2931 // 'method_result' is only used as output register at the very end of this method. 2932 // Until then we can reuse it as 'holder_offset'. 2933 Register holder_offset = method_result; 2934 assert_different_registers(resolved_klass, recv_klass, holder_klass, temp_itbl_klass, scan_temp, holder_offset); 2935 2936 int vtable_start_offset_bytes = in_bytes(Klass::vtable_start_offset()); 2937 int scan_step = itableOffsetEntry::size() * wordSize; 2938 int ioffset_bytes = in_bytes(itableOffsetEntry::interface_offset()); 2939 int ooffset_bytes = in_bytes(itableOffsetEntry::offset_offset()); 2940 int itmentry_off_bytes = in_bytes(itableMethodEntry::method_offset()); 2941 const int vte_scale = exact_log2(vtableEntry::size_in_bytes()); 2942 2943 Label L_loop_search_resolved_entry, L_resolved_found, L_holder_found; 2944 2945 lwu(scan_temp, Address(recv_klass, Klass::vtable_length_offset())); 2946 add(recv_klass, recv_klass, vtable_start_offset_bytes + ioffset_bytes); 2947 // itableOffsetEntry[] itable = recv_klass + Klass::vtable_start_offset() 2948 // + sizeof(vtableEntry) * (recv_klass->_vtable_len); 2949 // scan_temp = &(itable[0]._interface) 2950 // temp_itbl_klass = itable[0]._interface; 2951 shadd(scan_temp, scan_temp, recv_klass, scan_temp, vte_scale); 2952 ld(temp_itbl_klass, Address(scan_temp)); 2953 mv(holder_offset, zr); 2954 2955 // Initial checks: 2956 // - if (holder_klass != resolved_klass), go to "scan for resolved" 2957 // - if (itable[0] == holder_klass), shortcut to "holder found" 2958 // - if (itable[0] == 0), no such interface 2959 bne(resolved_klass, holder_klass, L_loop_search_resolved_entry); 2960 beq(holder_klass, temp_itbl_klass, L_holder_found); 2961 beqz(temp_itbl_klass, L_no_such_interface); 2962 2963 // Loop: Look for holder_klass record in itable 2964 // do { 2965 // temp_itbl_klass = *(scan_temp += scan_step); 2966 // if (temp_itbl_klass == holder_klass) { 2967 // goto L_holder_found; // Found! 2968 // } 2969 // } while (temp_itbl_klass != 0); 2970 // goto L_no_such_interface // Not found. 2971 Label L_search_holder; 2972 bind(L_search_holder); 2973 add(scan_temp, scan_temp, scan_step); 2974 ld(temp_itbl_klass, Address(scan_temp)); 2975 beq(holder_klass, temp_itbl_klass, L_holder_found); 2976 bnez(temp_itbl_klass, L_search_holder); 2977 2978 j(L_no_such_interface); 2979 2980 // Loop: Look for resolved_class record in itable 2981 // while (true) { 2982 // temp_itbl_klass = *(scan_temp += scan_step); 2983 // if (temp_itbl_klass == 0) { 2984 // goto L_no_such_interface; 2985 // } 2986 // if (temp_itbl_klass == resolved_klass) { 2987 // goto L_resolved_found; // Found! 2988 // } 2989 // if (temp_itbl_klass == holder_klass) { 2990 // holder_offset = scan_temp; 2991 // } 2992 // } 2993 // 2994 Label L_loop_search_resolved; 2995 bind(L_loop_search_resolved); 2996 add(scan_temp, scan_temp, scan_step); 2997 ld(temp_itbl_klass, Address(scan_temp)); 2998 bind(L_loop_search_resolved_entry); 2999 beqz(temp_itbl_klass, L_no_such_interface); 3000 beq(resolved_klass, temp_itbl_klass, L_resolved_found); 3001 bne(holder_klass, temp_itbl_klass, L_loop_search_resolved); 3002 mv(holder_offset, scan_temp); 3003 j(L_loop_search_resolved); 3004 3005 // See if we already have a holder klass. If not, go and scan for it. 3006 bind(L_resolved_found); 3007 beqz(holder_offset, L_search_holder); 3008 mv(scan_temp, holder_offset); 3009 3010 // Finally, scan_temp contains holder_klass vtable offset 3011 bind(L_holder_found); 3012 lwu(method_result, Address(scan_temp, ooffset_bytes - ioffset_bytes)); 3013 add(recv_klass, recv_klass, itable_index * wordSize + itmentry_off_bytes 3014 - vtable_start_offset_bytes - ioffset_bytes); // substract offsets to restore the original value of recv_klass 3015 add(method_result, recv_klass, method_result); 3016 ld(method_result, Address(method_result)); 3017 } 3018 3019 // virtual method calling 3020 void MacroAssembler::lookup_virtual_method(Register recv_klass, 3021 RegisterOrConstant vtable_index, 3022 Register method_result) { 3023 const ByteSize base = Klass::vtable_start_offset(); 3024 assert(vtableEntry::size() * wordSize == 8, 3025 "adjust the scaling in the code below"); 3026 int vtable_offset_in_bytes = in_bytes(base + vtableEntry::method_offset()); 3027 3028 if (vtable_index.is_register()) { 3029 shadd(method_result, vtable_index.as_register(), recv_klass, method_result, LogBytesPerWord); 3030 ld(method_result, Address(method_result, vtable_offset_in_bytes)); 3031 } else { 3032 vtable_offset_in_bytes += vtable_index.as_constant() * wordSize; 3033 ld(method_result, form_address(method_result, recv_klass, vtable_offset_in_bytes)); 3034 } 3035 } 3036 3037 void MacroAssembler::membar(uint32_t order_constraint) { 3038 address prev = pc() - MacroAssembler::instruction_size; 3039 address last = code()->last_insn(); 3040 3041 if (last != nullptr && is_membar(last) && prev == last) { 3042 // We are merging two memory barrier instructions. On RISCV we 3043 // can do this simply by ORing them together. 3044 set_membar_kind(prev, get_membar_kind(prev) | order_constraint); 3045 BLOCK_COMMENT("merged membar"); 3046 } else { 3047 code()->set_last_insn(pc()); 3048 3049 uint32_t predecessor = 0; 3050 uint32_t successor = 0; 3051 3052 membar_mask_to_pred_succ(order_constraint, predecessor, successor); 3053 fence(predecessor, successor); 3054 } 3055 } 3056 3057 // Form an address from base + offset in Rd. Rd my or may not 3058 // actually be used: you must use the Address that is returned. It 3059 // is up to you to ensure that the shift provided matches the size 3060 // of your data. 3061 Address MacroAssembler::form_address(Register Rd, Register base, int64_t byte_offset) { 3062 if (is_simm12(byte_offset)) { // 12: imm in range 2^12 3063 return Address(base, byte_offset); 3064 } 3065 3066 assert_different_registers(Rd, base, noreg); 3067 3068 // Do it the hard way 3069 mv(Rd, byte_offset); 3070 add(Rd, base, Rd); 3071 return Address(Rd); 3072 } 3073 3074 void MacroAssembler::check_klass_subtype(Register sub_klass, 3075 Register super_klass, 3076 Register tmp_reg, 3077 Label& L_success) { 3078 Label L_failure; 3079 check_klass_subtype_fast_path(sub_klass, super_klass, tmp_reg, &L_success, &L_failure, nullptr); 3080 check_klass_subtype_slow_path(sub_klass, super_klass, tmp_reg, noreg, &L_success, nullptr); 3081 bind(L_failure); 3082 } 3083 3084 void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod) { 3085 ld(t0, Address(xthread, JavaThread::polling_word_offset())); 3086 if (acquire) { 3087 membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); 3088 } 3089 if (at_return) { 3090 bgtu(in_nmethod ? sp : fp, t0, slow_path, /* is_far */ true); 3091 } else { 3092 test_bit(t0, t0, exact_log2(SafepointMechanism::poll_bit())); 3093 bnez(t0, slow_path, true /* is_far */); 3094 } 3095 } 3096 3097 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp, 3098 Label &succeed, Label *fail) { 3099 assert_different_registers(addr, tmp, t0); 3100 assert_different_registers(newv, tmp, t0); 3101 assert_different_registers(oldv, tmp, t0); 3102 3103 // oldv holds comparison value 3104 // newv holds value to write in exchange 3105 // addr identifies memory word to compare against/update 3106 if (UseZacas) { 3107 mv(tmp, oldv); 3108 atomic_cas(tmp, newv, addr, Assembler::int64, Assembler::aq, Assembler::rl); 3109 beq(tmp, oldv, succeed); 3110 } else { 3111 Label retry_load, nope; 3112 bind(retry_load); 3113 // Load reserved from the memory location 3114 load_reserved(tmp, addr, int64, Assembler::aqrl); 3115 // Fail and exit if it is not what we expect 3116 bne(tmp, oldv, nope); 3117 // If the store conditional succeeds, tmp will be zero 3118 store_conditional(tmp, newv, addr, int64, Assembler::rl); 3119 beqz(tmp, succeed); 3120 // Retry only when the store conditional failed 3121 j(retry_load); 3122 3123 bind(nope); 3124 } 3125 3126 // neither amocas nor lr/sc have an implied barrier in the failing case 3127 membar(AnyAny); 3128 3129 mv(oldv, tmp); 3130 if (fail != nullptr) { 3131 j(*fail); 3132 } 3133 } 3134 3135 void MacroAssembler::cmpxchg_obj_header(Register oldv, Register newv, Register obj, Register tmp, 3136 Label &succeed, Label *fail) { 3137 assert(oopDesc::mark_offset_in_bytes() == 0, "assumption"); 3138 cmpxchgptr(oldv, newv, obj, tmp, succeed, fail); 3139 } 3140 3141 void MacroAssembler::load_reserved(Register dst, 3142 Register addr, 3143 enum operand_size size, 3144 Assembler::Aqrl acquire) { 3145 switch (size) { 3146 case int64: 3147 lr_d(dst, addr, acquire); 3148 break; 3149 case int32: 3150 lr_w(dst, addr, acquire); 3151 break; 3152 case uint32: 3153 lr_w(dst, addr, acquire); 3154 zero_extend(dst, dst, 32); 3155 break; 3156 default: 3157 ShouldNotReachHere(); 3158 } 3159 } 3160 3161 void MacroAssembler::store_conditional(Register dst, 3162 Register new_val, 3163 Register addr, 3164 enum operand_size size, 3165 Assembler::Aqrl release) { 3166 switch (size) { 3167 case int64: 3168 sc_d(dst, new_val, addr, release); 3169 break; 3170 case int32: 3171 case uint32: 3172 sc_w(dst, new_val, addr, release); 3173 break; 3174 default: 3175 ShouldNotReachHere(); 3176 } 3177 } 3178 3179 3180 void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected, 3181 Register new_val, 3182 enum operand_size size, 3183 Register tmp1, Register tmp2, Register tmp3) { 3184 assert(size == int8 || size == int16, "unsupported operand size"); 3185 3186 Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3; 3187 3188 andi(shift, addr, 3); 3189 slli(shift, shift, 3); 3190 3191 andi(aligned_addr, addr, ~3); 3192 3193 if (size == int8) { 3194 mv(mask, 0xff); 3195 } else { 3196 // size == int16 case 3197 mv(mask, -1); 3198 zero_extend(mask, mask, 16); 3199 } 3200 sll(mask, mask, shift); 3201 3202 notr(not_mask, mask); 3203 3204 sll(expected, expected, shift); 3205 andr(expected, expected, mask); 3206 3207 sll(new_val, new_val, shift); 3208 andr(new_val, new_val, mask); 3209 } 3210 3211 // cmpxchg_narrow_value will kill t0, t1, expected, new_val and tmps. 3212 // It's designed to implement compare and swap byte/boolean/char/short by lr.w/sc.w or amocas.w, 3213 // which are forced to work with 4-byte aligned address. 3214 void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected, 3215 Register new_val, 3216 enum operand_size size, 3217 Assembler::Aqrl acquire, Assembler::Aqrl release, 3218 Register result, bool result_as_bool, 3219 Register tmp1, Register tmp2, Register tmp3) { 3220 Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0; 3221 assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp); 3222 cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3); 3223 3224 Label retry, fail, done; 3225 3226 bind(retry); 3227 3228 if (UseZacas) { 3229 lw(old, aligned_addr); 3230 3231 // if old & mask != expected 3232 andr(tmp, old, mask); 3233 bne(tmp, expected, fail); 3234 3235 andr(tmp, old, not_mask); 3236 orr(tmp, tmp, new_val); 3237 3238 atomic_cas(old, tmp, aligned_addr, operand_size::int32, acquire, release); 3239 bne(tmp, old, retry); 3240 } else { 3241 lr_w(old, aligned_addr, acquire); 3242 andr(tmp, old, mask); 3243 bne(tmp, expected, fail); 3244 3245 andr(tmp, old, not_mask); 3246 orr(tmp, tmp, new_val); 3247 sc_w(tmp, tmp, aligned_addr, release); 3248 bnez(tmp, retry); 3249 } 3250 3251 if (result_as_bool) { 3252 mv(result, 1); 3253 j(done); 3254 3255 bind(fail); 3256 mv(result, zr); 3257 3258 bind(done); 3259 } else { 3260 andr(tmp, old, mask); 3261 3262 bind(fail); 3263 srl(result, tmp, shift); 3264 3265 if (size == int8) { 3266 sign_extend(result, result, 8); 3267 } else { 3268 // size == int16 case 3269 sign_extend(result, result, 16); 3270 } 3271 } 3272 } 3273 3274 // weak_cmpxchg_narrow_value is a weak version of cmpxchg_narrow_value, to implement 3275 // the weak CAS stuff. The major difference is that it just failed when store conditional 3276 // failed. 3277 void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected, 3278 Register new_val, 3279 enum operand_size size, 3280 Assembler::Aqrl acquire, Assembler::Aqrl release, 3281 Register result, 3282 Register tmp1, Register tmp2, Register tmp3) { 3283 Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0; 3284 assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp); 3285 cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3); 3286 3287 Label fail, done; 3288 3289 if (UseZacas) { 3290 lw(old, aligned_addr); 3291 3292 // if old & mask != expected 3293 andr(tmp, old, mask); 3294 bne(tmp, expected, fail); 3295 3296 andr(tmp, old, not_mask); 3297 orr(tmp, tmp, new_val); 3298 3299 atomic_cas(tmp, new_val, addr, operand_size::int32, acquire, release); 3300 bne(tmp, old, fail); 3301 } else { 3302 lr_w(old, aligned_addr, acquire); 3303 andr(tmp, old, mask); 3304 bne(tmp, expected, fail); 3305 3306 andr(tmp, old, not_mask); 3307 orr(tmp, tmp, new_val); 3308 sc_w(tmp, tmp, aligned_addr, release); 3309 bnez(tmp, fail); 3310 } 3311 3312 // Success 3313 mv(result, 1); 3314 j(done); 3315 3316 // Fail 3317 bind(fail); 3318 mv(result, zr); 3319 3320 bind(done); 3321 } 3322 3323 void MacroAssembler::cmpxchg(Register addr, Register expected, 3324 Register new_val, 3325 enum operand_size size, 3326 Assembler::Aqrl acquire, Assembler::Aqrl release, 3327 Register result, bool result_as_bool) { 3328 assert(size != int8 && size != int16, "unsupported operand size"); 3329 assert_different_registers(addr, t0); 3330 assert_different_registers(expected, t0); 3331 assert_different_registers(new_val, t0); 3332 3333 if (UseZacas) { 3334 if (result_as_bool) { 3335 mv(t0, expected); 3336 atomic_cas(t0, new_val, addr, size, acquire, release); 3337 xorr(t0, t0, expected); 3338 seqz(result, t0); 3339 } else { 3340 mv(result, expected); 3341 atomic_cas(result, new_val, addr, size, acquire, release); 3342 } 3343 return; 3344 } 3345 3346 Label retry_load, done, ne_done; 3347 bind(retry_load); 3348 load_reserved(t0, addr, size, acquire); 3349 bne(t0, expected, ne_done); 3350 store_conditional(t0, new_val, addr, size, release); 3351 bnez(t0, retry_load); 3352 3353 // equal, succeed 3354 if (result_as_bool) { 3355 mv(result, 1); 3356 } else { 3357 mv(result, expected); 3358 } 3359 j(done); 3360 3361 // not equal, failed 3362 bind(ne_done); 3363 if (result_as_bool) { 3364 mv(result, zr); 3365 } else { 3366 mv(result, t0); 3367 } 3368 3369 bind(done); 3370 } 3371 3372 void MacroAssembler::cmpxchg_weak(Register addr, Register expected, 3373 Register new_val, 3374 enum operand_size size, 3375 Assembler::Aqrl acquire, Assembler::Aqrl release, 3376 Register result) { 3377 if (UseZacas) { 3378 cmpxchg(addr, expected, new_val, size, acquire, release, result, true); 3379 return; 3380 } 3381 3382 assert_different_registers(addr, t0); 3383 assert_different_registers(expected, t0); 3384 assert_different_registers(new_val, t0); 3385 3386 Label fail, done; 3387 load_reserved(t0, addr, size, acquire); 3388 bne(t0, expected, fail); 3389 store_conditional(t0, new_val, addr, size, release); 3390 bnez(t0, fail); 3391 3392 // Success 3393 mv(result, 1); 3394 j(done); 3395 3396 // Fail 3397 bind(fail); 3398 mv(result, zr); 3399 3400 bind(done); 3401 } 3402 3403 #define ATOMIC_OP(NAME, AOP, ACQUIRE, RELEASE) \ 3404 void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \ 3405 prev = prev->is_valid() ? prev : zr; \ 3406 if (incr.is_register()) { \ 3407 AOP(prev, addr, incr.as_register(), (Assembler::Aqrl)(ACQUIRE | RELEASE)); \ 3408 } else { \ 3409 mv(t0, incr.as_constant()); \ 3410 AOP(prev, addr, t0, (Assembler::Aqrl)(ACQUIRE | RELEASE)); \ 3411 } \ 3412 return; \ 3413 } 3414 3415 ATOMIC_OP(add, amoadd_d, Assembler::relaxed, Assembler::relaxed) 3416 ATOMIC_OP(addw, amoadd_w, Assembler::relaxed, Assembler::relaxed) 3417 ATOMIC_OP(addal, amoadd_d, Assembler::aq, Assembler::rl) 3418 ATOMIC_OP(addalw, amoadd_w, Assembler::aq, Assembler::rl) 3419 3420 #undef ATOMIC_OP 3421 3422 #define ATOMIC_XCHG(OP, AOP, ACQUIRE, RELEASE) \ 3423 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \ 3424 prev = prev->is_valid() ? prev : zr; \ 3425 AOP(prev, addr, newv, (Assembler::Aqrl)(ACQUIRE | RELEASE)); \ 3426 return; \ 3427 } 3428 3429 ATOMIC_XCHG(xchg, amoswap_d, Assembler::relaxed, Assembler::relaxed) 3430 ATOMIC_XCHG(xchgw, amoswap_w, Assembler::relaxed, Assembler::relaxed) 3431 ATOMIC_XCHG(xchgal, amoswap_d, Assembler::aq, Assembler::rl) 3432 ATOMIC_XCHG(xchgalw, amoswap_w, Assembler::aq, Assembler::rl) 3433 3434 #undef ATOMIC_XCHG 3435 3436 #define ATOMIC_XCHGU(OP1, OP2) \ 3437 void MacroAssembler::atomic_##OP1(Register prev, Register newv, Register addr) { \ 3438 atomic_##OP2(prev, newv, addr); \ 3439 zero_extend(prev, prev, 32); \ 3440 return; \ 3441 } 3442 3443 ATOMIC_XCHGU(xchgwu, xchgw) 3444 ATOMIC_XCHGU(xchgalwu, xchgalw) 3445 3446 #undef ATOMIC_XCHGU 3447 3448 #define ATOMIC_CAS(OP, AOP, ACQUIRE, RELEASE) \ 3449 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \ 3450 assert(UseZacas, "invariant"); \ 3451 prev = prev->is_valid() ? prev : zr; \ 3452 AOP(prev, addr, newv, (Assembler::Aqrl)(ACQUIRE | RELEASE)); \ 3453 return; \ 3454 } 3455 3456 ATOMIC_CAS(cas, amocas_d, Assembler::relaxed, Assembler::relaxed) 3457 ATOMIC_CAS(casw, amocas_w, Assembler::relaxed, Assembler::relaxed) 3458 ATOMIC_CAS(casl, amocas_d, Assembler::relaxed, Assembler::rl) 3459 ATOMIC_CAS(caslw, amocas_w, Assembler::relaxed, Assembler::rl) 3460 ATOMIC_CAS(casal, amocas_d, Assembler::aq, Assembler::rl) 3461 ATOMIC_CAS(casalw, amocas_w, Assembler::aq, Assembler::rl) 3462 3463 #undef ATOMIC_CAS 3464 3465 #define ATOMIC_CASU(OP1, OP2) \ 3466 void MacroAssembler::atomic_##OP1(Register prev, Register newv, Register addr) { \ 3467 atomic_##OP2(prev, newv, addr); \ 3468 zero_extend(prev, prev, 32); \ 3469 return; \ 3470 } 3471 3472 ATOMIC_CASU(caswu, casw) 3473 ATOMIC_CASU(caslwu, caslw) 3474 ATOMIC_CASU(casalwu, casalw) 3475 3476 #undef ATOMIC_CASU 3477 3478 void MacroAssembler::atomic_cas( 3479 Register prev, Register newv, Register addr, enum operand_size size, Assembler::Aqrl acquire, Assembler::Aqrl release) { 3480 switch (size) { 3481 case int64: 3482 switch ((Assembler::Aqrl)(acquire | release)) { 3483 case Assembler::relaxed: 3484 atomic_cas(prev, newv, addr); 3485 break; 3486 case Assembler::rl: 3487 atomic_casl(prev, newv, addr); 3488 break; 3489 case Assembler::aqrl: 3490 atomic_casal(prev, newv, addr); 3491 break; 3492 default: 3493 ShouldNotReachHere(); 3494 } 3495 break; 3496 case int32: 3497 switch ((Assembler::Aqrl)(acquire | release)) { 3498 case Assembler::relaxed: 3499 atomic_casw(prev, newv, addr); 3500 break; 3501 case Assembler::rl: 3502 atomic_caslw(prev, newv, addr); 3503 break; 3504 case Assembler::aqrl: 3505 atomic_casalw(prev, newv, addr); 3506 break; 3507 default: 3508 ShouldNotReachHere(); 3509 } 3510 break; 3511 case uint32: 3512 switch ((Assembler::Aqrl)(acquire | release)) { 3513 case Assembler::relaxed: 3514 atomic_caswu(prev, newv, addr); 3515 break; 3516 case Assembler::rl: 3517 atomic_caslwu(prev, newv, addr); 3518 break; 3519 case Assembler::aqrl: 3520 atomic_casalwu(prev, newv, addr); 3521 break; 3522 default: 3523 ShouldNotReachHere(); 3524 } 3525 break; 3526 default: 3527 ShouldNotReachHere(); 3528 } 3529 } 3530 3531 void MacroAssembler::far_jump(const Address &entry, Register tmp) { 3532 assert(CodeCache::find_blob(entry.target()) != nullptr, 3533 "destination of far jump not found in code cache"); 3534 assert(entry.rspec().type() == relocInfo::external_word_type 3535 || entry.rspec().type() == relocInfo::runtime_call_type 3536 || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type"); 3537 // Fixed length: see MacroAssembler::far_branch_size() 3538 // We can use auipc + jr here because we know that the total size of 3539 // the code cache cannot exceed 2Gb. 3540 relocate(entry.rspec(), [&] { 3541 int64_t distance = entry.target() - pc(); 3542 int32_t offset = ((int32_t)distance << 20) >> 20; 3543 assert(is_valid_32bit_offset(distance), "Far jump using wrong instructions."); 3544 auipc(tmp, (int32_t)distance + 0x800); 3545 jr(tmp, offset); 3546 }); 3547 } 3548 3549 void MacroAssembler::far_call(const Address &entry, Register tmp) { 3550 assert(CodeCache::find_blob(entry.target()) != nullptr, 3551 "destination of far call not found in code cache"); 3552 assert(entry.rspec().type() == relocInfo::external_word_type 3553 || entry.rspec().type() == relocInfo::runtime_call_type 3554 || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type"); 3555 // Fixed length: see MacroAssembler::far_branch_size() 3556 // We can use auipc + jalr here because we know that the total size of 3557 // the code cache cannot exceed 2Gb. 3558 relocate(entry.rspec(), [&] { 3559 int64_t distance = entry.target() - pc(); 3560 int32_t offset = ((int32_t)distance << 20) >> 20; 3561 assert(is_valid_32bit_offset(distance), "Far call using wrong instructions."); 3562 auipc(tmp, (int32_t)distance + 0x800); 3563 jalr(tmp, offset); 3564 }); 3565 } 3566 3567 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, 3568 Register super_klass, 3569 Register tmp_reg, 3570 Label* L_success, 3571 Label* L_failure, 3572 Label* L_slow_path, 3573 Register super_check_offset) { 3574 assert_different_registers(sub_klass, super_klass, tmp_reg); 3575 bool must_load_sco = (super_check_offset == noreg); 3576 if (must_load_sco) { 3577 assert(tmp_reg != noreg, "supply either a temp or a register offset"); 3578 } else { 3579 assert_different_registers(sub_klass, super_klass, super_check_offset); 3580 } 3581 3582 Label L_fallthrough; 3583 int label_nulls = 0; 3584 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 3585 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 3586 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; } 3587 assert(label_nulls <= 1, "at most one null in batch"); 3588 3589 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 3590 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 3591 Address super_check_offset_addr(super_klass, sco_offset); 3592 3593 // Hacked jmp, which may only be used just before L_fallthrough. 3594 #define final_jmp(label) \ 3595 if (&(label) == &L_fallthrough) { /*do nothing*/ } \ 3596 else j(label) /*omit semi*/ 3597 3598 // If the pointers are equal, we are done (e.g., String[] elements). 3599 // This self-check enables sharing of secondary supertype arrays among 3600 // non-primary types such as array-of-interface. Otherwise, each such 3601 // type would need its own customized SSA. 3602 // We move this check to the front of the fast path because many 3603 // type checks are in fact trivially successful in this manner, 3604 // so we get a nicely predicted branch right at the start of the check. 3605 beq(sub_klass, super_klass, *L_success); 3606 3607 // Check the supertype display: 3608 if (must_load_sco) { 3609 lwu(tmp_reg, super_check_offset_addr); 3610 super_check_offset = tmp_reg; 3611 } 3612 add(t0, sub_klass, super_check_offset); 3613 Address super_check_addr(t0); 3614 ld(t0, super_check_addr); // load displayed supertype 3615 3616 // This check has worked decisively for primary supers. 3617 // Secondary supers are sought in the super_cache ('super_cache_addr'). 3618 // (Secondary supers are interfaces and very deeply nested subtypes.) 3619 // This works in the same check above because of a tricky aliasing 3620 // between the super_Cache and the primary super display elements. 3621 // (The 'super_check_addr' can address either, as the case requires.) 3622 // Note that the cache is updated below if it does not help us find 3623 // what we need immediately. 3624 // So if it was a primary super, we can just fail immediately. 3625 // Otherwise, it's the slow path for us (no success at this point). 3626 3627 beq(super_klass, t0, *L_success); 3628 mv(t1, sc_offset); 3629 if (L_failure == &L_fallthrough) { 3630 beq(super_check_offset, t1, *L_slow_path); 3631 } else { 3632 bne(super_check_offset, t1, *L_failure, /* is_far */ true); 3633 final_jmp(*L_slow_path); 3634 } 3635 3636 bind(L_fallthrough); 3637 3638 #undef final_jmp 3639 } 3640 3641 // Scans count pointer sized words at [addr] for occurrence of value, 3642 // generic 3643 void MacroAssembler::repne_scan(Register addr, Register value, Register count, 3644 Register tmp) { 3645 Label Lloop, Lexit; 3646 beqz(count, Lexit); 3647 bind(Lloop); 3648 ld(tmp, addr); 3649 beq(value, tmp, Lexit); 3650 add(addr, addr, wordSize); 3651 sub(count, count, 1); 3652 bnez(count, Lloop); 3653 bind(Lexit); 3654 } 3655 3656 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 3657 Register super_klass, 3658 Register tmp1_reg, 3659 Register tmp2_reg, 3660 Label* L_success, 3661 Label* L_failure) { 3662 assert_different_registers(sub_klass, super_klass, tmp1_reg); 3663 if (tmp2_reg != noreg) { 3664 assert_different_registers(sub_klass, super_klass, tmp1_reg, tmp2_reg, t0); 3665 } 3666 #define IS_A_TEMP(reg) ((reg) == tmp1_reg || (reg) == tmp2_reg) 3667 3668 Label L_fallthrough; 3669 int label_nulls = 0; 3670 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 3671 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 3672 3673 assert(label_nulls <= 1, "at most one null in the batch"); 3674 3675 // A couple of useful fields in sub_klass: 3676 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 3677 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 3678 Address secondary_supers_addr(sub_klass, ss_offset); 3679 Address super_cache_addr( sub_klass, sc_offset); 3680 3681 BLOCK_COMMENT("check_klass_subtype_slow_path"); 3682 3683 // Do a linear scan of the secondary super-klass chain. 3684 // This code is rarely used, so simplicity is a virtue here. 3685 // The repne_scan instruction uses fixed registers, which we must spill. 3686 // Don't worry too much about pre-existing connections with the input regs. 3687 3688 assert(sub_klass != x10, "killed reg"); // killed by mv(x10, super) 3689 assert(sub_klass != x12, "killed reg"); // killed by la(x12, &pst_counter) 3690 3691 RegSet pushed_registers; 3692 if (!IS_A_TEMP(x12)) { 3693 pushed_registers += x12; 3694 } 3695 if (!IS_A_TEMP(x15)) { 3696 pushed_registers += x15; 3697 } 3698 3699 if (super_klass != x10) { 3700 if (!IS_A_TEMP(x10)) { 3701 pushed_registers += x10; 3702 } 3703 } 3704 3705 push_reg(pushed_registers, sp); 3706 3707 // Get super_klass value into x10 (even if it was in x15 or x12) 3708 mv(x10, super_klass); 3709 3710 #ifndef PRODUCT 3711 incrementw(ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr)); 3712 #endif // PRODUCT 3713 3714 // We will consult the secondary-super array. 3715 ld(x15, secondary_supers_addr); 3716 // Load the array length. 3717 lwu(x12, Address(x15, Array<Klass*>::length_offset_in_bytes())); 3718 // Skip to start of data. 3719 add(x15, x15, Array<Klass*>::base_offset_in_bytes()); 3720 3721 // Set t0 to an obvious invalid value, falling through by default 3722 mv(t0, -1); 3723 // Scan X12 words at [X15] for an occurrence of X10. 3724 repne_scan(x15, x10, x12, t0); 3725 3726 // pop will restore x10, so we should use a temp register to keep its value 3727 mv(t1, x10); 3728 3729 // Unspill the temp registers: 3730 pop_reg(pushed_registers, sp); 3731 3732 bne(t1, t0, *L_failure); 3733 3734 // Success. Cache the super we found an proceed in triumph. 3735 sd(super_klass, super_cache_addr); 3736 3737 if (L_success != &L_fallthrough) { 3738 j(*L_success); 3739 } 3740 3741 #undef IS_A_TEMP 3742 3743 bind(L_fallthrough); 3744 } 3745 3746 // population_count variant for running without the CPOP 3747 // instruction, which was introduced with Zbb extension. 3748 void MacroAssembler::population_count(Register dst, Register src, 3749 Register tmp1, Register tmp2) { 3750 if (UsePopCountInstruction) { 3751 cpop(dst, src); 3752 } else { 3753 assert_different_registers(src, tmp1, tmp2); 3754 assert_different_registers(dst, tmp1, tmp2); 3755 Label loop, done; 3756 3757 mv(tmp1, src); 3758 // dst = 0; 3759 // while(tmp1 != 0) { 3760 // dst++; 3761 // tmp1 &= (tmp1 - 1); 3762 // } 3763 mv(dst, zr); 3764 beqz(tmp1, done); 3765 { 3766 bind(loop); 3767 addi(dst, dst, 1); 3768 addi(tmp2, tmp1, -1); 3769 andr(tmp1, tmp1, tmp2); 3770 bnez(tmp1, loop); 3771 } 3772 bind(done); 3773 } 3774 } 3775 3776 // Ensure that the inline code and the stub are using the same registers 3777 // as we need to call the stub from inline code when there is a collision 3778 // in the hashed lookup in the secondary supers array. 3779 #define LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS(r_super_klass, r_array_base, r_array_length, \ 3780 r_array_index, r_sub_klass, result, r_bitmap) \ 3781 do { \ 3782 assert(r_super_klass == x10 && \ 3783 r_array_base == x11 && \ 3784 r_array_length == x12 && \ 3785 (r_array_index == x13 || r_array_index == noreg) && \ 3786 (r_sub_klass == x14 || r_sub_klass == noreg) && \ 3787 (result == x15 || result == noreg) && \ 3788 (r_bitmap == x16 || r_bitmap == noreg), "registers must match riscv.ad"); \ 3789 } while(0) 3790 3791 // Return true: we succeeded in generating this code 3792 bool MacroAssembler::lookup_secondary_supers_table(Register r_sub_klass, 3793 Register r_super_klass, 3794 Register result, 3795 Register tmp1, 3796 Register tmp2, 3797 Register tmp3, 3798 Register tmp4, 3799 u1 super_klass_slot, 3800 bool stub_is_near) { 3801 assert_different_registers(r_sub_klass, r_super_klass, result, tmp1, tmp2, tmp3, tmp4, t0); 3802 3803 Label L_fallthrough; 3804 3805 BLOCK_COMMENT("lookup_secondary_supers_table {"); 3806 3807 const Register 3808 r_array_base = tmp1, // x11 3809 r_array_length = tmp2, // x12 3810 r_array_index = tmp3, // x13 3811 r_bitmap = tmp4; // x16 3812 3813 LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS(r_super_klass, r_array_base, r_array_length, 3814 r_array_index, r_sub_klass, result, r_bitmap); 3815 3816 u1 bit = super_klass_slot; 3817 3818 // Initialize result value to 1 which means mismatch. 3819 mv(result, 1); 3820 3821 ld(r_bitmap, Address(r_sub_klass, Klass::bitmap_offset())); 3822 3823 // First check the bitmap to see if super_klass might be present. If 3824 // the bit is zero, we are certain that super_klass is not one of 3825 // the secondary supers. 3826 test_bit(t0, r_bitmap, bit); 3827 beqz(t0, L_fallthrough); 3828 3829 // Get the first array index that can contain super_klass into r_array_index. 3830 if (bit != 0) { 3831 slli(r_array_index, r_bitmap, (Klass::SECONDARY_SUPERS_TABLE_MASK - bit)); 3832 population_count(r_array_index, r_array_index, tmp1, tmp2); 3833 } else { 3834 mv(r_array_index, (u1)1); 3835 } 3836 3837 // We will consult the secondary-super array. 3838 ld(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 3839 3840 // The value i in r_array_index is >= 1, so even though r_array_base 3841 // points to the length, we don't need to adjust it to point to the data. 3842 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, "Adjust this code"); 3843 assert(Array<Klass*>::length_offset_in_bytes() == 0, "Adjust this code"); 3844 3845 shadd(result, r_array_index, r_array_base, result, LogBytesPerWord); 3846 ld(result, Address(result)); 3847 xorr(result, result, r_super_klass); 3848 beqz(result, L_fallthrough); // Found a match 3849 3850 // Is there another entry to check? Consult the bitmap. 3851 test_bit(t0, r_bitmap, (bit + 1) & Klass::SECONDARY_SUPERS_TABLE_MASK); 3852 beqz(t0, L_fallthrough); 3853 3854 // Linear probe. 3855 if (bit != 0) { 3856 ror_imm(r_bitmap, r_bitmap, bit); 3857 } 3858 3859 // The slot we just inspected is at secondary_supers[r_array_index - 1]. 3860 // The next slot to be inspected, by the stub we're about to call, 3861 // is secondary_supers[r_array_index]. Bits 0 and 1 in the bitmap 3862 // have been checked. 3863 rt_call(StubRoutines::lookup_secondary_supers_table_slow_path_stub()); 3864 3865 BLOCK_COMMENT("} lookup_secondary_supers_table"); 3866 3867 bind(L_fallthrough); 3868 3869 if (VerifySecondarySupers) { 3870 verify_secondary_supers_table(r_sub_klass, r_super_klass, // x14, x10 3871 result, tmp1, tmp2, tmp3); // x15, x11, x12, x13 3872 } 3873 return true; 3874 } 3875 3876 // Called by code generated by check_klass_subtype_slow_path 3877 // above. This is called when there is a collision in the hashed 3878 // lookup in the secondary supers array. 3879 void MacroAssembler::lookup_secondary_supers_table_slow_path(Register r_super_klass, 3880 Register r_array_base, 3881 Register r_array_index, 3882 Register r_bitmap, 3883 Register result, 3884 Register tmp1) { 3885 assert_different_registers(r_super_klass, r_array_base, r_array_index, r_bitmap, tmp1, result, t0); 3886 3887 const Register 3888 r_array_length = tmp1, 3889 r_sub_klass = noreg; // unused 3890 3891 LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS(r_super_klass, r_array_base, r_array_length, 3892 r_array_index, r_sub_klass, result, r_bitmap); 3893 3894 Label L_matched, L_fallthrough, L_bitmap_full; 3895 3896 // Initialize result value to 1 which means mismatch. 3897 mv(result, 1); 3898 3899 // Load the array length. 3900 lwu(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes())); 3901 // And adjust the array base to point to the data. 3902 // NB! Effectively increments current slot index by 1. 3903 assert(Array<Klass*>::base_offset_in_bytes() == wordSize, ""); 3904 addi(r_array_base, r_array_base, Array<Klass*>::base_offset_in_bytes()); 3905 3906 // Check if bitmap is SECONDARY_SUPERS_BITMAP_FULL 3907 assert(Klass::SECONDARY_SUPERS_BITMAP_FULL == ~uintx(0), "Adjust this code"); 3908 subw(t0, r_array_length, Klass::SECONDARY_SUPERS_TABLE_SIZE - 2); 3909 bgtz(t0, L_bitmap_full); 3910 3911 // NB! Our caller has checked bits 0 and 1 in the bitmap. The 3912 // current slot (at secondary_supers[r_array_index]) has not yet 3913 // been inspected, and r_array_index may be out of bounds if we 3914 // wrapped around the end of the array. 3915 3916 { // This is conventional linear probing, but instead of terminating 3917 // when a null entry is found in the table, we maintain a bitmap 3918 // in which a 0 indicates missing entries. 3919 // The check above guarantees there are 0s in the bitmap, so the loop 3920 // eventually terminates. 3921 Label L_loop; 3922 bind(L_loop); 3923 3924 // Check for wraparound. 3925 Label skip; 3926 blt(r_array_index, r_array_length, skip); 3927 mv(r_array_index, zr); 3928 bind(skip); 3929 3930 shadd(t0, r_array_index, r_array_base, t0, LogBytesPerWord); 3931 ld(t0, Address(t0)); 3932 beq(t0, r_super_klass, L_matched); 3933 3934 test_bit(t0, r_bitmap, 2); // look-ahead check (Bit 2); result is non-zero 3935 beqz(t0, L_fallthrough); 3936 3937 ror_imm(r_bitmap, r_bitmap, 1); 3938 addi(r_array_index, r_array_index, 1); 3939 j(L_loop); 3940 } 3941 3942 { // Degenerate case: more than 64 secondary supers. 3943 // FIXME: We could do something smarter here, maybe a vectorized 3944 // comparison or a binary search, but is that worth any added 3945 // complexity? 3946 bind(L_bitmap_full); 3947 repne_scan(r_array_base, r_super_klass, r_array_length, t0); 3948 bne(r_super_klass, t0, L_fallthrough); 3949 } 3950 3951 bind(L_matched); 3952 mv(result, zr); 3953 3954 bind(L_fallthrough); 3955 } 3956 3957 // Make sure that the hashed lookup and a linear scan agree. 3958 void MacroAssembler::verify_secondary_supers_table(Register r_sub_klass, 3959 Register r_super_klass, 3960 Register result, 3961 Register tmp1, 3962 Register tmp2, 3963 Register tmp3) { 3964 assert_different_registers(r_sub_klass, r_super_klass, tmp1, tmp2, tmp3, result, t0); 3965 3966 const Register 3967 r_array_base = tmp1, // X11 3968 r_array_length = tmp2, // X12 3969 r_array_index = noreg, // unused 3970 r_bitmap = noreg; // unused 3971 3972 LOOKUP_SECONDARY_SUPERS_TABLE_REGISTERS(r_super_klass, r_array_base, r_array_length, 3973 r_array_index, r_sub_klass, result, r_bitmap); 3974 3975 BLOCK_COMMENT("verify_secondary_supers_table {"); 3976 3977 // We will consult the secondary-super array. 3978 ld(r_array_base, Address(r_sub_klass, in_bytes(Klass::secondary_supers_offset()))); 3979 3980 // Load the array length. 3981 lwu(r_array_length, Address(r_array_base, Array<Klass*>::length_offset_in_bytes())); 3982 // And adjust the array base to point to the data. 3983 addi(r_array_base, r_array_base, Array<Klass*>::base_offset_in_bytes()); 3984 3985 repne_scan(r_array_base, r_super_klass, r_array_length, t0); 3986 Label failed; 3987 mv(tmp3, 1); 3988 bne(r_super_klass, t0, failed); 3989 mv(tmp3, zr); 3990 bind(failed); 3991 3992 snez(result, result); // normalize result to 0/1 for comparison 3993 3994 Label passed; 3995 beq(tmp3, result, passed); 3996 { 3997 mv(x10, r_super_klass); 3998 mv(x11, r_sub_klass); 3999 mv(x12, tmp3); 4000 mv(x13, result); 4001 mv(x14, (address)("mismatch")); 4002 rt_call(CAST_FROM_FN_PTR(address, Klass::on_secondary_supers_verification_failure)); 4003 should_not_reach_here(); 4004 } 4005 bind(passed); 4006 4007 BLOCK_COMMENT("} verify_secondary_supers_table"); 4008 } 4009 4010 // Defines obj, preserves var_size_in_bytes, okay for tmp2 == var_size_in_bytes. 4011 void MacroAssembler::tlab_allocate(Register obj, 4012 Register var_size_in_bytes, 4013 int con_size_in_bytes, 4014 Register tmp1, 4015 Register tmp2, 4016 Label& slow_case, 4017 bool is_far) { 4018 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 4019 bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, tmp1, tmp2, slow_case, is_far); 4020 } 4021 4022 // get_thread() can be called anywhere inside generated code so we 4023 // need to save whatever non-callee save context might get clobbered 4024 // by the call to Thread::current() or, indeed, the call setup code. 4025 void MacroAssembler::get_thread(Register thread) { 4026 // save all call-clobbered regs except thread 4027 RegSet saved_regs = RegSet::range(x5, x7) + RegSet::range(x10, x17) + 4028 RegSet::range(x28, x31) + ra - thread; 4029 push_reg(saved_regs, sp); 4030 4031 mv(ra, CAST_FROM_FN_PTR(address, Thread::current)); 4032 jalr(ra); 4033 if (thread != c_rarg0) { 4034 mv(thread, c_rarg0); 4035 } 4036 4037 // restore pushed registers 4038 pop_reg(saved_regs, sp); 4039 } 4040 4041 void MacroAssembler::load_byte_map_base(Register reg) { 4042 CardTable::CardValue* byte_map_base = 4043 ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base(); 4044 mv(reg, (uint64_t)byte_map_base); 4045 } 4046 4047 void MacroAssembler::build_frame(int framesize) { 4048 assert(framesize >= 2, "framesize must include space for FP/RA"); 4049 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment"); 4050 sub(sp, sp, framesize); 4051 sd(fp, Address(sp, framesize - 2 * wordSize)); 4052 sd(ra, Address(sp, framesize - wordSize)); 4053 if (PreserveFramePointer) { add(fp, sp, framesize); } 4054 } 4055 4056 void MacroAssembler::remove_frame(int framesize) { 4057 assert(framesize >= 2, "framesize must include space for FP/RA"); 4058 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment"); 4059 ld(fp, Address(sp, framesize - 2 * wordSize)); 4060 ld(ra, Address(sp, framesize - wordSize)); 4061 add(sp, sp, framesize); 4062 } 4063 4064 void MacroAssembler::reserved_stack_check() { 4065 // testing if reserved zone needs to be enabled 4066 Label no_reserved_zone_enabling; 4067 4068 ld(t0, Address(xthread, JavaThread::reserved_stack_activation_offset())); 4069 bltu(sp, t0, no_reserved_zone_enabling); 4070 4071 enter(); // RA and FP are live. 4072 mv(c_rarg0, xthread); 4073 rt_call(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone)); 4074 leave(); 4075 4076 // We have already removed our own frame. 4077 // throw_delayed_StackOverflowError will think that it's been 4078 // called by our caller. 4079 la(t0, RuntimeAddress(SharedRuntime::throw_delayed_StackOverflowError_entry())); 4080 jr(t0); 4081 should_not_reach_here(); 4082 4083 bind(no_reserved_zone_enabling); 4084 } 4085 4086 // Move the address of the polling page into dest. 4087 void MacroAssembler::get_polling_page(Register dest, relocInfo::relocType rtype) { 4088 ld(dest, Address(xthread, JavaThread::polling_page_offset())); 4089 } 4090 4091 // Read the polling page. The address of the polling page must 4092 // already be in r. 4093 void MacroAssembler::read_polling_page(Register r, int32_t offset, relocInfo::relocType rtype) { 4094 relocate(rtype, [&] { 4095 lwu(zr, Address(r, offset)); 4096 }); 4097 } 4098 4099 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { 4100 #ifdef ASSERT 4101 { 4102 ThreadInVMfromUnknown tiv; 4103 assert (UseCompressedOops, "should only be used for compressed oops"); 4104 assert (Universe::heap() != nullptr, "java heap should be initialized"); 4105 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 4106 assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop"); 4107 } 4108 #endif 4109 int oop_index = oop_recorder()->find_index(obj); 4110 relocate(oop_Relocation::spec(oop_index), [&] { 4111 li32(dst, 0xDEADBEEF); 4112 }); 4113 zero_extend(dst, dst, 32); 4114 } 4115 4116 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { 4117 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 4118 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 4119 int index = oop_recorder()->find_index(k); 4120 assert(!Universe::heap()->is_in(k), "should not be an oop"); 4121 4122 narrowKlass nk = CompressedKlassPointers::encode(k); 4123 relocate(metadata_Relocation::spec(index), [&] { 4124 li32(dst, nk); 4125 }); 4126 zero_extend(dst, dst, 32); 4127 } 4128 4129 // Maybe emit a call via a trampoline. If the code cache is small 4130 // trampolines won't be emitted. 4131 address MacroAssembler::trampoline_call(Address entry) { 4132 assert(entry.rspec().type() == relocInfo::runtime_call_type || 4133 entry.rspec().type() == relocInfo::opt_virtual_call_type || 4134 entry.rspec().type() == relocInfo::static_call_type || 4135 entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type"); 4136 4137 address target = entry.target(); 4138 4139 // We need a trampoline if branches are far. 4140 if (!in_scratch_emit_size()) { 4141 if (entry.rspec().type() == relocInfo::runtime_call_type) { 4142 assert(CodeBuffer::supports_shared_stubs(), "must support shared stubs"); 4143 code()->share_trampoline_for(entry.target(), offset()); 4144 } else { 4145 address stub = emit_trampoline_stub(offset(), target); 4146 if (stub == nullptr) { 4147 postcond(pc() == badAddress); 4148 return nullptr; // CodeCache is full 4149 } 4150 } 4151 } 4152 target = pc(); 4153 4154 address call_pc = pc(); 4155 #ifdef ASSERT 4156 if (entry.rspec().type() != relocInfo::runtime_call_type) { 4157 assert_alignment(call_pc); 4158 } 4159 #endif 4160 relocate(entry.rspec(), [&] { 4161 jump_link(target, t0); 4162 }); 4163 4164 postcond(pc() != badAddress); 4165 return call_pc; 4166 } 4167 4168 address MacroAssembler::load_and_call(Address entry) { 4169 assert(entry.rspec().type() == relocInfo::runtime_call_type || 4170 entry.rspec().type() == relocInfo::opt_virtual_call_type || 4171 entry.rspec().type() == relocInfo::static_call_type || 4172 entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type"); 4173 4174 address target = entry.target(); 4175 4176 if (!in_scratch_emit_size()) { 4177 address stub = emit_address_stub(offset(), target); 4178 if (stub == nullptr) { 4179 postcond(pc() == badAddress); 4180 return nullptr; // CodeCache is full 4181 } 4182 } 4183 4184 address call_pc = pc(); 4185 #ifdef ASSERT 4186 if (entry.rspec().type() != relocInfo::runtime_call_type) { 4187 assert_alignment(call_pc); 4188 } 4189 #endif 4190 relocate(entry.rspec(), [&] { 4191 load_link_jump(target); 4192 }); 4193 4194 postcond(pc() != badAddress); 4195 return call_pc; 4196 } 4197 4198 address MacroAssembler::ic_call(address entry, jint method_index) { 4199 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); 4200 IncompressibleRegion ir(this); // relocations 4201 movptr(t1, (address)Universe::non_oop_word(), t0); 4202 assert_cond(entry != nullptr); 4203 return reloc_call(Address(entry, rh)); 4204 } 4205 4206 int MacroAssembler::ic_check_size() { 4207 // No compressed 4208 return (MacroAssembler::instruction_size * (2 /* 2 loads */ + 1 /* branch */)) + 4209 far_branch_size(); 4210 } 4211 4212 int MacroAssembler::ic_check(int end_alignment) { 4213 IncompressibleRegion ir(this); 4214 Register receiver = j_rarg0; 4215 Register data = t1; 4216 4217 Register tmp1 = t0; // t0 always scratch 4218 // t2 is saved on call, thus should have been saved before this check. 4219 // Hence we can clobber it. 4220 Register tmp2 = t2; 4221 4222 // The UEP of a code blob ensures that the VEP is padded. However, the padding of the UEP is placed 4223 // before the inline cache check, so we don't have to execute any nop instructions when dispatching 4224 // through the UEP, yet we can ensure that the VEP is aligned appropriately. That's why we align 4225 // before the inline cache check here, and not after 4226 align(end_alignment, ic_check_size()); 4227 int uep_offset = offset(); 4228 4229 if (UseCompressedClassPointers) { 4230 lwu(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes())); 4231 lwu(tmp2, Address(data, CompiledICData::speculated_klass_offset())); 4232 } else { 4233 ld(tmp1, Address(receiver, oopDesc::klass_offset_in_bytes())); 4234 ld(tmp2, Address(data, CompiledICData::speculated_klass_offset())); 4235 } 4236 4237 Label ic_hit; 4238 beq(tmp1, tmp2, ic_hit); 4239 // Note, far_jump is not fixed size. 4240 // Is this ever generates a movptr alignment/size will be off. 4241 far_jump(RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 4242 bind(ic_hit); 4243 4244 assert((offset() % end_alignment) == 0, "Misaligned verified entry point."); 4245 return uep_offset; 4246 } 4247 4248 address MacroAssembler::emit_address_stub(int insts_call_instruction_offset, address dest) { 4249 address stub = start_a_stub(max_reloc_call_stub_size()); 4250 if (stub == nullptr) { 4251 return nullptr; // CodeBuffer::expand failed 4252 } 4253 4254 // We are always 4-byte aligned here. 4255 assert_alignment(pc()); 4256 4257 // Make sure the address of destination 8-byte aligned. 4258 align(wordSize, 0); 4259 4260 RelocationHolder rh = trampoline_stub_Relocation::spec(code()->insts()->start() + 4261 insts_call_instruction_offset); 4262 const int stub_start_offset = offset(); 4263 relocate(rh, [&] { 4264 assert(offset() - stub_start_offset == 0, 4265 "%ld - %ld == %ld : should be", (long)offset(), (long)stub_start_offset, (long)0); 4266 assert(offset() % wordSize == 0, "bad alignment"); 4267 emit_int64((int64_t)dest); 4268 }); 4269 4270 const address stub_start_addr = addr_at(stub_start_offset); 4271 end_a_stub(); 4272 4273 return stub_start_addr; 4274 } 4275 4276 // Emit a trampoline stub for a call to a target which is too far away. 4277 // 4278 // code sequences: 4279 // 4280 // call-site: 4281 // branch-and-link to <destination> or <trampoline stub> 4282 // 4283 // Related trampoline stub for this call site in the stub section: 4284 // load the call target from the constant pool 4285 // branch (RA still points to the call site above) 4286 4287 address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, 4288 address dest) { 4289 // Max stub size: alignment nop, TrampolineStub. 4290 address stub = start_a_stub(max_reloc_call_stub_size()); 4291 if (stub == nullptr) { 4292 return nullptr; // CodeBuffer::expand failed 4293 } 4294 4295 assert(UseTrampolines, "Must be using trampos."); 4296 4297 // We are always 4-byte aligned here. 4298 assert_alignment(pc()); 4299 4300 // Create a trampoline stub relocation which relates this trampoline stub 4301 // with the call instruction at insts_call_instruction_offset in the 4302 // instructions code-section. 4303 4304 // Make sure the address of destination 8-byte aligned after 3 instructions. 4305 align(wordSize, MacroAssembler::NativeShortCall::trampoline_data_offset); 4306 4307 RelocationHolder rh = trampoline_stub_Relocation::spec(code()->insts()->start() + 4308 insts_call_instruction_offset); 4309 const int stub_start_offset = offset(); 4310 relocate(rh, [&] { 4311 // Now, create the trampoline stub's code: 4312 // - load the call 4313 // - call 4314 Label target; 4315 ld(t0, target); // auipc + ld 4316 jr(t0); // jalr 4317 bind(target); 4318 assert(offset() - stub_start_offset == MacroAssembler::NativeShortCall::trampoline_data_offset, 4319 "should be"); 4320 assert(offset() % wordSize == 0, "bad alignment"); 4321 emit_int64((int64_t)dest); 4322 }); 4323 4324 const address stub_start_addr = addr_at(stub_start_offset); 4325 4326 end_a_stub(); 4327 4328 return stub_start_addr; 4329 } 4330 4331 int MacroAssembler::max_reloc_call_stub_size() { 4332 // Max stub size: alignment nop, TrampolineStub. 4333 if (UseTrampolines) { 4334 return instruction_size + MacroAssembler::NativeShortCall::trampoline_size; 4335 } 4336 return instruction_size + wordSize; 4337 } 4338 4339 int MacroAssembler::static_call_stub_size() { 4340 // (lui, addi, slli, addi, slli, addi) + (lui + lui + slli + add) + jalr 4341 return 11 * MacroAssembler::instruction_size; 4342 } 4343 4344 Address MacroAssembler::add_memory_helper(const Address dst, Register tmp) { 4345 switch (dst.getMode()) { 4346 case Address::base_plus_offset: 4347 // This is the expected mode, although we allow all the other 4348 // forms below. 4349 return form_address(tmp, dst.base(), dst.offset()); 4350 default: 4351 la(tmp, dst); 4352 return Address(tmp); 4353 } 4354 } 4355 4356 void MacroAssembler::increment(const Address dst, int64_t value, Register tmp1, Register tmp2) { 4357 assert(((dst.getMode() == Address::base_plus_offset && 4358 is_simm12(dst.offset())) || is_simm12(value)), 4359 "invalid value and address mode combination"); 4360 Address adr = add_memory_helper(dst, tmp2); 4361 assert(!adr.uses(tmp1), "invalid dst for address increment"); 4362 ld(tmp1, adr); 4363 add(tmp1, tmp1, value, tmp2); 4364 sd(tmp1, adr); 4365 } 4366 4367 void MacroAssembler::incrementw(const Address dst, int32_t value, Register tmp1, Register tmp2) { 4368 assert(((dst.getMode() == Address::base_plus_offset && 4369 is_simm12(dst.offset())) || is_simm12(value)), 4370 "invalid value and address mode combination"); 4371 Address adr = add_memory_helper(dst, tmp2); 4372 assert(!adr.uses(tmp1), "invalid dst for address increment"); 4373 lwu(tmp1, adr); 4374 addw(tmp1, tmp1, value, tmp2); 4375 sw(tmp1, adr); 4376 } 4377 4378 void MacroAssembler::decrement(const Address dst, int64_t value, Register tmp1, Register tmp2) { 4379 assert(((dst.getMode() == Address::base_plus_offset && 4380 is_simm12(dst.offset())) || is_simm12(value)), 4381 "invalid value and address mode combination"); 4382 Address adr = add_memory_helper(dst, tmp2); 4383 assert(!adr.uses(tmp1), "invalid dst for address decrement"); 4384 ld(tmp1, adr); 4385 sub(tmp1, tmp1, value, tmp2); 4386 sd(tmp1, adr); 4387 } 4388 4389 void MacroAssembler::decrementw(const Address dst, int32_t value, Register tmp1, Register tmp2) { 4390 assert(((dst.getMode() == Address::base_plus_offset && 4391 is_simm12(dst.offset())) || is_simm12(value)), 4392 "invalid value and address mode combination"); 4393 Address adr = add_memory_helper(dst, tmp2); 4394 assert(!adr.uses(tmp1), "invalid dst for address decrement"); 4395 lwu(tmp1, adr); 4396 subw(tmp1, tmp1, value, tmp2); 4397 sw(tmp1, adr); 4398 } 4399 4400 void MacroAssembler::cmpptr(Register src1, Address src2, Label& equal) { 4401 assert_different_registers(src1, t0); 4402 relocate(src2.rspec(), [&] { 4403 int32_t offset; 4404 la(t0, src2.target(), offset); 4405 ld(t0, Address(t0, offset)); 4406 }); 4407 beq(src1, t0, equal); 4408 } 4409 4410 void MacroAssembler::load_method_holder_cld(Register result, Register method) { 4411 load_method_holder(result, method); 4412 ld(result, Address(result, InstanceKlass::class_loader_data_offset())); 4413 } 4414 4415 void MacroAssembler::load_method_holder(Register holder, Register method) { 4416 ld(holder, Address(method, Method::const_offset())); // ConstMethod* 4417 ld(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* 4418 ld(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* 4419 } 4420 4421 // string indexof 4422 // compute index by trailing zeros 4423 void MacroAssembler::compute_index(Register haystack, Register trailing_zeros, 4424 Register match_mask, Register result, 4425 Register ch2, Register tmp, 4426 bool haystack_isL) { 4427 int haystack_chr_shift = haystack_isL ? 0 : 1; 4428 srl(match_mask, match_mask, trailing_zeros); 4429 srli(match_mask, match_mask, 1); 4430 srli(tmp, trailing_zeros, LogBitsPerByte); 4431 if (!haystack_isL) andi(tmp, tmp, 0xE); 4432 add(haystack, haystack, tmp); 4433 ld(ch2, Address(haystack)); 4434 if (!haystack_isL) srli(tmp, tmp, haystack_chr_shift); 4435 add(result, result, tmp); 4436 } 4437 4438 // string indexof 4439 // Find pattern element in src, compute match mask, 4440 // only the first occurrence of 0x80/0x8000 at low bits is the valid match index 4441 // match mask patterns and corresponding indices would be like: 4442 // - 0x8080808080808080 (Latin1) 4443 // - 7 6 5 4 3 2 1 0 (match index) 4444 // - 0x8000800080008000 (UTF16) 4445 // - 3 2 1 0 (match index) 4446 void MacroAssembler::compute_match_mask(Register src, Register pattern, Register match_mask, 4447 Register mask1, Register mask2) { 4448 xorr(src, pattern, src); 4449 sub(match_mask, src, mask1); 4450 orr(src, src, mask2); 4451 notr(src, src); 4452 andr(match_mask, match_mask, src); 4453 } 4454 4455 #ifdef COMPILER2 4456 // Code for BigInteger::mulAdd intrinsic 4457 // out = x10 4458 // in = x11 4459 // offset = x12 (already out.length-offset) 4460 // len = x13 4461 // k = x14 4462 // tmp = x28 4463 // 4464 // pseudo code from java implementation: 4465 // long kLong = k & LONG_MASK; 4466 // carry = 0; 4467 // offset = out.length-offset - 1; 4468 // for (int j = len - 1; j >= 0; j--) { 4469 // product = (in[j] & LONG_MASK) * kLong + (out[offset] & LONG_MASK) + carry; 4470 // out[offset--] = (int)product; 4471 // carry = product >>> 32; 4472 // } 4473 // return (int)carry; 4474 void MacroAssembler::mul_add(Register out, Register in, Register offset, 4475 Register len, Register k, Register tmp) { 4476 Label L_tail_loop, L_unroll, L_end; 4477 mv(tmp, out); 4478 mv(out, zr); 4479 blez(len, L_end); 4480 zero_extend(k, k, 32); 4481 slliw(t0, offset, LogBytesPerInt); 4482 add(offset, tmp, t0); 4483 slliw(t0, len, LogBytesPerInt); 4484 add(in, in, t0); 4485 4486 const int unroll = 8; 4487 mv(tmp, unroll); 4488 blt(len, tmp, L_tail_loop); 4489 bind(L_unroll); 4490 for (int i = 0; i < unroll; i++) { 4491 sub(in, in, BytesPerInt); 4492 lwu(t0, Address(in, 0)); 4493 mul(t1, t0, k); 4494 add(t0, t1, out); 4495 sub(offset, offset, BytesPerInt); 4496 lwu(t1, Address(offset, 0)); 4497 add(t0, t0, t1); 4498 sw(t0, Address(offset, 0)); 4499 srli(out, t0, 32); 4500 } 4501 subw(len, len, tmp); 4502 bge(len, tmp, L_unroll); 4503 4504 bind(L_tail_loop); 4505 blez(len, L_end); 4506 sub(in, in, BytesPerInt); 4507 lwu(t0, Address(in, 0)); 4508 mul(t1, t0, k); 4509 add(t0, t1, out); 4510 sub(offset, offset, BytesPerInt); 4511 lwu(t1, Address(offset, 0)); 4512 add(t0, t0, t1); 4513 sw(t0, Address(offset, 0)); 4514 srli(out, t0, 32); 4515 subw(len, len, 1); 4516 j(L_tail_loop); 4517 4518 bind(L_end); 4519 } 4520 4521 // Multiply and multiply-accumulate unsigned 64-bit registers. 4522 void MacroAssembler::wide_mul(Register prod_lo, Register prod_hi, Register n, Register m) { 4523 assert_different_registers(prod_lo, prod_hi); 4524 4525 mul(prod_lo, n, m); 4526 mulhu(prod_hi, n, m); 4527 } 4528 4529 void MacroAssembler::wide_madd(Register sum_lo, Register sum_hi, Register n, 4530 Register m, Register tmp1, Register tmp2) { 4531 assert_different_registers(sum_lo, sum_hi); 4532 assert_different_registers(sum_hi, tmp2); 4533 4534 wide_mul(tmp1, tmp2, n, m); 4535 cad(sum_lo, sum_lo, tmp1, tmp1); // Add tmp1 to sum_lo with carry output to tmp1 4536 adc(sum_hi, sum_hi, tmp2, tmp1); // Add tmp2 with carry to sum_hi 4537 } 4538 4539 // add two unsigned input and output carry 4540 void MacroAssembler::cad(Register dst, Register src1, Register src2, Register carry) 4541 { 4542 assert_different_registers(dst, carry); 4543 assert_different_registers(dst, src2); 4544 add(dst, src1, src2); 4545 sltu(carry, dst, src2); 4546 } 4547 4548 // add two input with carry 4549 void MacroAssembler::adc(Register dst, Register src1, Register src2, Register carry) { 4550 assert_different_registers(dst, carry); 4551 add(dst, src1, src2); 4552 add(dst, dst, carry); 4553 } 4554 4555 // add two unsigned input with carry and output carry 4556 void MacroAssembler::cadc(Register dst, Register src1, Register src2, Register carry) { 4557 assert_different_registers(dst, src2); 4558 adc(dst, src1, src2, carry); 4559 sltu(carry, dst, src2); 4560 } 4561 4562 void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo, 4563 Register src1, Register src2, Register carry) { 4564 cad(dest_lo, dest_lo, src1, carry); 4565 add(dest_hi, dest_hi, carry); 4566 cad(dest_lo, dest_lo, src2, carry); 4567 add(final_dest_hi, dest_hi, carry); 4568 } 4569 4570 /** 4571 * Multiply 32 bit by 32 bit first loop. 4572 */ 4573 void MacroAssembler::multiply_32_x_32_loop(Register x, Register xstart, Register x_xstart, 4574 Register y, Register y_idx, Register z, 4575 Register carry, Register product, 4576 Register idx, Register kdx) { 4577 // jlong carry, x[], y[], z[]; 4578 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) { 4579 // long product = y[idx] * x[xstart] + carry; 4580 // z[kdx] = (int)product; 4581 // carry = product >>> 32; 4582 // } 4583 // z[xstart] = (int)carry; 4584 4585 Label L_first_loop, L_first_loop_exit; 4586 blez(idx, L_first_loop_exit); 4587 4588 shadd(t0, xstart, x, t0, LogBytesPerInt); 4589 lwu(x_xstart, Address(t0, 0)); 4590 4591 bind(L_first_loop); 4592 subw(idx, idx, 1); 4593 shadd(t0, idx, y, t0, LogBytesPerInt); 4594 lwu(y_idx, Address(t0, 0)); 4595 mul(product, x_xstart, y_idx); 4596 add(product, product, carry); 4597 srli(carry, product, 32); 4598 subw(kdx, kdx, 1); 4599 shadd(t0, kdx, z, t0, LogBytesPerInt); 4600 sw(product, Address(t0, 0)); 4601 bgtz(idx, L_first_loop); 4602 4603 bind(L_first_loop_exit); 4604 } 4605 4606 /** 4607 * Multiply 64 bit by 64 bit first loop. 4608 */ 4609 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, 4610 Register y, Register y_idx, Register z, 4611 Register carry, Register product, 4612 Register idx, Register kdx) { 4613 // 4614 // jlong carry, x[], y[], z[]; 4615 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) { 4616 // huge_128 product = y[idx] * x[xstart] + carry; 4617 // z[kdx] = (jlong)product; 4618 // carry = (jlong)(product >>> 64); 4619 // } 4620 // z[xstart] = carry; 4621 // 4622 4623 Label L_first_loop, L_first_loop_exit; 4624 Label L_one_x, L_one_y, L_multiply; 4625 4626 subw(xstart, xstart, 1); 4627 bltz(xstart, L_one_x); 4628 4629 shadd(t0, xstart, x, t0, LogBytesPerInt); 4630 ld(x_xstart, Address(t0, 0)); 4631 ror_imm(x_xstart, x_xstart, 32); // convert big-endian to little-endian 4632 4633 bind(L_first_loop); 4634 subw(idx, idx, 1); 4635 bltz(idx, L_first_loop_exit); 4636 subw(idx, idx, 1); 4637 bltz(idx, L_one_y); 4638 4639 shadd(t0, idx, y, t0, LogBytesPerInt); 4640 ld(y_idx, Address(t0, 0)); 4641 ror_imm(y_idx, y_idx, 32); // convert big-endian to little-endian 4642 bind(L_multiply); 4643 4644 mulhu(t0, x_xstart, y_idx); 4645 mul(product, x_xstart, y_idx); 4646 cad(product, product, carry, t1); 4647 adc(carry, t0, zr, t1); 4648 4649 subw(kdx, kdx, 2); 4650 ror_imm(product, product, 32); // back to big-endian 4651 shadd(t0, kdx, z, t0, LogBytesPerInt); 4652 sd(product, Address(t0, 0)); 4653 4654 j(L_first_loop); 4655 4656 bind(L_one_y); 4657 lwu(y_idx, Address(y, 0)); 4658 j(L_multiply); 4659 4660 bind(L_one_x); 4661 lwu(x_xstart, Address(x, 0)); 4662 j(L_first_loop); 4663 4664 bind(L_first_loop_exit); 4665 } 4666 4667 /** 4668 * Multiply 128 bit by 128 bit. Unrolled inner loop. 4669 * 4670 */ 4671 void MacroAssembler::multiply_128_x_128_loop(Register y, Register z, 4672 Register carry, Register carry2, 4673 Register idx, Register jdx, 4674 Register yz_idx1, Register yz_idx2, 4675 Register tmp, Register tmp3, Register tmp4, 4676 Register tmp6, Register product_hi) { 4677 // jlong carry, x[], y[], z[]; 4678 // int kdx = xstart+1; 4679 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 4680 // huge_128 tmp3 = (y[idx+1] * product_hi) + z[kdx+idx+1] + carry; 4681 // jlong carry2 = (jlong)(tmp3 >>> 64); 4682 // huge_128 tmp4 = (y[idx] * product_hi) + z[kdx+idx] + carry2; 4683 // carry = (jlong)(tmp4 >>> 64); 4684 // z[kdx+idx+1] = (jlong)tmp3; 4685 // z[kdx+idx] = (jlong)tmp4; 4686 // } 4687 // idx += 2; 4688 // if (idx > 0) { 4689 // yz_idx1 = (y[idx] * product_hi) + z[kdx+idx] + carry; 4690 // z[kdx+idx] = (jlong)yz_idx1; 4691 // carry = (jlong)(yz_idx1 >>> 64); 4692 // } 4693 // 4694 4695 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 4696 4697 srliw(jdx, idx, 2); 4698 4699 bind(L_third_loop); 4700 4701 subw(jdx, jdx, 1); 4702 bltz(jdx, L_third_loop_exit); 4703 subw(idx, idx, 4); 4704 4705 shadd(t0, idx, y, t0, LogBytesPerInt); 4706 ld(yz_idx2, Address(t0, 0)); 4707 ld(yz_idx1, Address(t0, wordSize)); 4708 4709 shadd(tmp6, idx, z, t0, LogBytesPerInt); 4710 4711 ror_imm(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian 4712 ror_imm(yz_idx2, yz_idx2, 32); 4713 4714 ld(t1, Address(tmp6, 0)); 4715 ld(t0, Address(tmp6, wordSize)); 4716 4717 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3 4718 mulhu(tmp4, product_hi, yz_idx1); 4719 4720 ror_imm(t0, t0, 32, tmp); // convert big-endian to little-endian 4721 ror_imm(t1, t1, 32, tmp); 4722 4723 mul(tmp, product_hi, yz_idx2); // yz_idx2 * product_hi -> carry2:tmp 4724 mulhu(carry2, product_hi, yz_idx2); 4725 4726 cad(tmp3, tmp3, carry, carry); 4727 adc(tmp4, tmp4, zr, carry); 4728 cad(tmp3, tmp3, t0, t0); 4729 cadc(tmp4, tmp4, tmp, t0); 4730 adc(carry, carry2, zr, t0); 4731 cad(tmp4, tmp4, t1, carry2); 4732 adc(carry, carry, zr, carry2); 4733 4734 ror_imm(tmp3, tmp3, 32); // convert little-endian to big-endian 4735 ror_imm(tmp4, tmp4, 32); 4736 sd(tmp4, Address(tmp6, 0)); 4737 sd(tmp3, Address(tmp6, wordSize)); 4738 4739 j(L_third_loop); 4740 4741 bind(L_third_loop_exit); 4742 4743 andi(idx, idx, 0x3); 4744 beqz(idx, L_post_third_loop_done); 4745 4746 Label L_check_1; 4747 subw(idx, idx, 2); 4748 bltz(idx, L_check_1); 4749 4750 shadd(t0, idx, y, t0, LogBytesPerInt); 4751 ld(yz_idx1, Address(t0, 0)); 4752 ror_imm(yz_idx1, yz_idx1, 32); 4753 4754 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3 4755 mulhu(tmp4, product_hi, yz_idx1); 4756 4757 shadd(t0, idx, z, t0, LogBytesPerInt); 4758 ld(yz_idx2, Address(t0, 0)); 4759 ror_imm(yz_idx2, yz_idx2, 32, tmp); 4760 4761 add2_with_carry(carry, tmp4, tmp3, carry, yz_idx2, tmp); 4762 4763 ror_imm(tmp3, tmp3, 32, tmp); 4764 sd(tmp3, Address(t0, 0)); 4765 4766 bind(L_check_1); 4767 4768 andi(idx, idx, 0x1); 4769 subw(idx, idx, 1); 4770 bltz(idx, L_post_third_loop_done); 4771 shadd(t0, idx, y, t0, LogBytesPerInt); 4772 lwu(tmp4, Address(t0, 0)); 4773 mul(tmp3, tmp4, product_hi); // tmp4 * product_hi -> carry2:tmp3 4774 mulhu(carry2, tmp4, product_hi); 4775 4776 shadd(t0, idx, z, t0, LogBytesPerInt); 4777 lwu(tmp4, Address(t0, 0)); 4778 4779 add2_with_carry(carry2, carry2, tmp3, tmp4, carry, t0); 4780 4781 shadd(t0, idx, z, t0, LogBytesPerInt); 4782 sw(tmp3, Address(t0, 0)); 4783 4784 slli(t0, carry2, 32); 4785 srli(carry, tmp3, 32); 4786 orr(carry, carry, t0); 4787 4788 bind(L_post_third_loop_done); 4789 } 4790 4791 /** 4792 * Code for BigInteger::multiplyToLen() intrinsic. 4793 * 4794 * x10: x 4795 * x11: xlen 4796 * x12: y 4797 * x13: ylen 4798 * x14: z 4799 * x15: tmp0 4800 * x16: tmp1 4801 * x17: tmp2 4802 * x7: tmp3 4803 * x28: tmp4 4804 * x29: tmp5 4805 * x30: tmp6 4806 * x31: tmp7 4807 */ 4808 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, 4809 Register z, Register tmp0, 4810 Register tmp1, Register tmp2, Register tmp3, Register tmp4, 4811 Register tmp5, Register tmp6, Register product_hi) { 4812 assert_different_registers(x, xlen, y, ylen, z, tmp0, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6); 4813 4814 const Register idx = tmp1; 4815 const Register kdx = tmp2; 4816 const Register xstart = tmp3; 4817 4818 const Register y_idx = tmp4; 4819 const Register carry = tmp5; 4820 const Register product = xlen; 4821 const Register x_xstart = tmp0; 4822 4823 mv(idx, ylen); // idx = ylen; 4824 addw(kdx, xlen, ylen); // kdx = xlen+ylen; 4825 mv(carry, zr); // carry = 0; 4826 4827 Label L_multiply_64_x_64_loop, L_done; 4828 4829 subw(xstart, xlen, 1); 4830 bltz(xstart, L_done); 4831 4832 const Register jdx = tmp1; 4833 4834 if (AvoidUnalignedAccesses) { 4835 // Check if x and y are both 8-byte aligned. 4836 orr(t0, xlen, ylen); 4837 test_bit(t0, t0, 0); 4838 beqz(t0, L_multiply_64_x_64_loop); 4839 4840 multiply_32_x_32_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 4841 shadd(t0, xstart, z, t0, LogBytesPerInt); 4842 sw(carry, Address(t0, 0)); 4843 4844 Label L_second_loop_unaligned; 4845 bind(L_second_loop_unaligned); 4846 mv(carry, zr); 4847 mv(jdx, ylen); 4848 subw(xstart, xstart, 1); 4849 bltz(xstart, L_done); 4850 sub(sp, sp, 2 * wordSize); 4851 sd(z, Address(sp, 0)); 4852 sd(zr, Address(sp, wordSize)); 4853 shadd(t0, xstart, z, t0, LogBytesPerInt); 4854 addi(z, t0, 4); 4855 shadd(t0, xstart, x, t0, LogBytesPerInt); 4856 lwu(product, Address(t0, 0)); 4857 Label L_third_loop, L_third_loop_exit; 4858 4859 blez(jdx, L_third_loop_exit); 4860 4861 bind(L_third_loop); 4862 subw(jdx, jdx, 1); 4863 shadd(t0, jdx, y, t0, LogBytesPerInt); 4864 lwu(t0, Address(t0, 0)); 4865 mul(t1, t0, product); 4866 add(t0, t1, carry); 4867 shadd(tmp6, jdx, z, t1, LogBytesPerInt); 4868 lwu(t1, Address(tmp6, 0)); 4869 add(t0, t0, t1); 4870 sw(t0, Address(tmp6, 0)); 4871 srli(carry, t0, 32); 4872 bgtz(jdx, L_third_loop); 4873 4874 bind(L_third_loop_exit); 4875 ld(z, Address(sp, 0)); 4876 addi(sp, sp, 2 * wordSize); 4877 shadd(t0, xstart, z, t0, LogBytesPerInt); 4878 sw(carry, Address(t0, 0)); 4879 4880 j(L_second_loop_unaligned); 4881 } 4882 4883 bind(L_multiply_64_x_64_loop); 4884 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 4885 4886 Label L_second_loop_aligned; 4887 beqz(kdx, L_second_loop_aligned); 4888 4889 Label L_carry; 4890 subw(kdx, kdx, 1); 4891 beqz(kdx, L_carry); 4892 4893 shadd(t0, kdx, z, t0, LogBytesPerInt); 4894 sw(carry, Address(t0, 0)); 4895 srli(carry, carry, 32); 4896 subw(kdx, kdx, 1); 4897 4898 bind(L_carry); 4899 shadd(t0, kdx, z, t0, LogBytesPerInt); 4900 sw(carry, Address(t0, 0)); 4901 4902 // Second and third (nested) loops. 4903 // 4904 // for (int i = xstart-1; i >= 0; i--) { // Second loop 4905 // carry = 0; 4906 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop 4907 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + 4908 // (z[k] & LONG_MASK) + carry; 4909 // z[k] = (int)product; 4910 // carry = product >>> 32; 4911 // } 4912 // z[i] = (int)carry; 4913 // } 4914 // 4915 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = product_hi 4916 4917 bind(L_second_loop_aligned); 4918 mv(carry, zr); // carry = 0; 4919 mv(jdx, ylen); // j = ystart+1 4920 4921 subw(xstart, xstart, 1); // i = xstart-1; 4922 bltz(xstart, L_done); 4923 4924 sub(sp, sp, 4 * wordSize); 4925 sd(z, Address(sp, 0)); 4926 4927 Label L_last_x; 4928 shadd(t0, xstart, z, t0, LogBytesPerInt); 4929 addi(z, t0, 4); 4930 subw(xstart, xstart, 1); // i = xstart-1; 4931 bltz(xstart, L_last_x); 4932 4933 shadd(t0, xstart, x, t0, LogBytesPerInt); 4934 ld(product_hi, Address(t0, 0)); 4935 ror_imm(product_hi, product_hi, 32); // convert big-endian to little-endian 4936 4937 Label L_third_loop_prologue; 4938 bind(L_third_loop_prologue); 4939 4940 sd(ylen, Address(sp, wordSize)); 4941 sd(x, Address(sp, 2 * wordSize)); 4942 sd(xstart, Address(sp, 3 * wordSize)); 4943 multiply_128_x_128_loop(y, z, carry, x, jdx, ylen, product, 4944 tmp2, x_xstart, tmp3, tmp4, tmp6, product_hi); 4945 ld(z, Address(sp, 0)); 4946 ld(ylen, Address(sp, wordSize)); 4947 ld(x, Address(sp, 2 * wordSize)); 4948 ld(xlen, Address(sp, 3 * wordSize)); // copy old xstart -> xlen 4949 addi(sp, sp, 4 * wordSize); 4950 4951 addiw(tmp3, xlen, 1); 4952 shadd(t0, tmp3, z, t0, LogBytesPerInt); 4953 sw(carry, Address(t0, 0)); 4954 4955 subw(tmp3, tmp3, 1); 4956 bltz(tmp3, L_done); 4957 4958 srli(carry, carry, 32); 4959 shadd(t0, tmp3, z, t0, LogBytesPerInt); 4960 sw(carry, Address(t0, 0)); 4961 j(L_second_loop_aligned); 4962 4963 // Next infrequent code is moved outside loops. 4964 bind(L_last_x); 4965 lwu(product_hi, Address(x, 0)); 4966 j(L_third_loop_prologue); 4967 4968 bind(L_done); 4969 } 4970 #endif 4971 4972 // Count bits of trailing zero chars from lsb to msb until first non-zero element. 4973 // For LL case, one byte for one element, so shift 8 bits once, and for other case, 4974 // shift 16 bits once. 4975 void MacroAssembler::ctzc_bit(Register Rd, Register Rs, bool isLL, Register tmp1, Register tmp2) { 4976 if (UseZbb) { 4977 assert_different_registers(Rd, Rs, tmp1); 4978 int step = isLL ? 8 : 16; 4979 ctz(Rd, Rs); 4980 andi(tmp1, Rd, step - 1); 4981 sub(Rd, Rd, tmp1); 4982 return; 4983 } 4984 4985 assert_different_registers(Rd, Rs, tmp1, tmp2); 4986 Label Loop; 4987 int step = isLL ? 8 : 16; 4988 mv(Rd, -step); 4989 mv(tmp2, Rs); 4990 4991 bind(Loop); 4992 addi(Rd, Rd, step); 4993 andi(tmp1, tmp2, ((1 << step) - 1)); 4994 srli(tmp2, tmp2, step); 4995 beqz(tmp1, Loop); 4996 } 4997 4998 // This instruction reads adjacent 4 bytes from the lower half of source register, 4999 // inflate into a register, for example: 5000 // Rs: A7A6A5A4A3A2A1A0 5001 // Rd: 00A300A200A100A0 5002 void MacroAssembler::inflate_lo32(Register Rd, Register Rs, Register tmp1, Register tmp2) { 5003 assert_different_registers(Rd, Rs, tmp1, tmp2); 5004 5005 mv(tmp1, 0xFF000000); // first byte mask at lower word 5006 andr(Rd, Rs, tmp1); 5007 for (int i = 0; i < 2; i++) { 5008 slli(Rd, Rd, wordSize); 5009 srli(tmp1, tmp1, wordSize); 5010 andr(tmp2, Rs, tmp1); 5011 orr(Rd, Rd, tmp2); 5012 } 5013 slli(Rd, Rd, wordSize); 5014 andi(tmp2, Rs, 0xFF); // last byte mask at lower word 5015 orr(Rd, Rd, tmp2); 5016 } 5017 5018 // This instruction reads adjacent 4 bytes from the upper half of source register, 5019 // inflate into a register, for example: 5020 // Rs: A7A6A5A4A3A2A1A0 5021 // Rd: 00A700A600A500A4 5022 void MacroAssembler::inflate_hi32(Register Rd, Register Rs, Register tmp1, Register tmp2) { 5023 assert_different_registers(Rd, Rs, tmp1, tmp2); 5024 srli(Rs, Rs, 32); // only upper 32 bits are needed 5025 inflate_lo32(Rd, Rs, tmp1, tmp2); 5026 } 5027 5028 // The size of the blocks erased by the zero_blocks stub. We must 5029 // handle anything smaller than this ourselves in zero_words(). 5030 const int MacroAssembler::zero_words_block_size = 8; 5031 5032 // zero_words() is used by C2 ClearArray patterns. It is as small as 5033 // possible, handling small word counts locally and delegating 5034 // anything larger to the zero_blocks stub. It is expanded many times 5035 // in compiled code, so it is important to keep it short. 5036 5037 // ptr: Address of a buffer to be zeroed. 5038 // cnt: Count in HeapWords. 5039 // 5040 // ptr, cnt, and t0 are clobbered. 5041 address MacroAssembler::zero_words(Register ptr, Register cnt) { 5042 assert(is_power_of_2(zero_words_block_size), "adjust this"); 5043 assert(ptr == x28 && cnt == x29, "mismatch in register usage"); 5044 assert_different_registers(cnt, t0); 5045 5046 BLOCK_COMMENT("zero_words {"); 5047 5048 mv(t0, zero_words_block_size); 5049 Label around, done, done16; 5050 bltu(cnt, t0, around); 5051 { 5052 RuntimeAddress zero_blocks(StubRoutines::riscv::zero_blocks()); 5053 assert(zero_blocks.target() != nullptr, "zero_blocks stub has not been generated"); 5054 if (StubRoutines::riscv::complete()) { 5055 address tpc = reloc_call(zero_blocks); 5056 if (tpc == nullptr) { 5057 DEBUG_ONLY(reset_labels(around)); 5058 postcond(pc() == badAddress); 5059 return nullptr; 5060 } 5061 } else { 5062 rt_call(zero_blocks.target()); 5063 } 5064 } 5065 bind(around); 5066 for (int i = zero_words_block_size >> 1; i > 1; i >>= 1) { 5067 Label l; 5068 test_bit(t0, cnt, exact_log2(i)); 5069 beqz(t0, l); 5070 for (int j = 0; j < i; j++) { 5071 sd(zr, Address(ptr, j * wordSize)); 5072 } 5073 addi(ptr, ptr, i * wordSize); 5074 bind(l); 5075 } 5076 { 5077 Label l; 5078 test_bit(t0, cnt, 0); 5079 beqz(t0, l); 5080 sd(zr, Address(ptr, 0)); 5081 bind(l); 5082 } 5083 5084 BLOCK_COMMENT("} zero_words"); 5085 postcond(pc() != badAddress); 5086 return pc(); 5087 } 5088 5089 #define SmallArraySize (18 * BytesPerLong) 5090 5091 // base: Address of a buffer to be zeroed, 8 bytes aligned. 5092 // cnt: Immediate count in HeapWords. 5093 void MacroAssembler::zero_words(Register base, uint64_t cnt) { 5094 assert_different_registers(base, t0, t1); 5095 5096 BLOCK_COMMENT("zero_words {"); 5097 5098 if (cnt <= SmallArraySize / BytesPerLong) { 5099 for (int i = 0; i < (int)cnt; i++) { 5100 sd(zr, Address(base, i * wordSize)); 5101 } 5102 } else { 5103 const int unroll = 8; // Number of sd(zr, adr), instructions we'll unroll 5104 int remainder = cnt % unroll; 5105 for (int i = 0; i < remainder; i++) { 5106 sd(zr, Address(base, i * wordSize)); 5107 } 5108 5109 Label loop; 5110 Register cnt_reg = t0; 5111 Register loop_base = t1; 5112 cnt = cnt - remainder; 5113 mv(cnt_reg, cnt); 5114 add(loop_base, base, remainder * wordSize); 5115 bind(loop); 5116 sub(cnt_reg, cnt_reg, unroll); 5117 for (int i = 0; i < unroll; i++) { 5118 sd(zr, Address(loop_base, i * wordSize)); 5119 } 5120 add(loop_base, loop_base, unroll * wordSize); 5121 bnez(cnt_reg, loop); 5122 } 5123 5124 BLOCK_COMMENT("} zero_words"); 5125 } 5126 5127 // base: Address of a buffer to be filled, 8 bytes aligned. 5128 // cnt: Count in 8-byte unit. 5129 // value: Value to be filled with. 5130 // base will point to the end of the buffer after filling. 5131 void MacroAssembler::fill_words(Register base, Register cnt, Register value) { 5132 // Algorithm: 5133 // 5134 // t0 = cnt & 7 5135 // cnt -= t0 5136 // p += t0 5137 // switch (t0): 5138 // switch start: 5139 // do while cnt 5140 // cnt -= 8 5141 // p[-8] = value 5142 // case 7: 5143 // p[-7] = value 5144 // case 6: 5145 // p[-6] = value 5146 // // ... 5147 // case 1: 5148 // p[-1] = value 5149 // case 0: 5150 // p += 8 5151 // do-while end 5152 // switch end 5153 5154 assert_different_registers(base, cnt, value, t0, t1); 5155 5156 Label fini, skip, entry, loop; 5157 const int unroll = 8; // Number of sd instructions we'll unroll 5158 5159 beqz(cnt, fini); 5160 5161 andi(t0, cnt, unroll - 1); 5162 sub(cnt, cnt, t0); 5163 // align 8, so first sd n % 8 = mod, next loop sd 8 * n. 5164 shadd(base, t0, base, t1, 3); 5165 la(t1, entry); 5166 slli(t0, t0, 2); // sd_inst_nums * 4; t0 is cnt % 8, so t1 = t1 - sd_inst_nums * 4, 4 is sizeof(inst) 5167 sub(t1, t1, t0); 5168 jr(t1); 5169 5170 bind(loop); 5171 add(base, base, unroll * 8); 5172 for (int i = -unroll; i < 0; i++) { 5173 sd(value, Address(base, i * 8)); 5174 } 5175 bind(entry); 5176 sub(cnt, cnt, unroll); 5177 bgez(cnt, loop); 5178 5179 bind(fini); 5180 } 5181 5182 // Zero blocks of memory by using CBO.ZERO. 5183 // 5184 // Aligns the base address first sufficiently for CBO.ZERO, then uses 5185 // CBO.ZERO repeatedly for every full block. cnt is the size to be 5186 // zeroed in HeapWords. Returns the count of words left to be zeroed 5187 // in cnt. 5188 // 5189 // NOTE: This is intended to be used in the zero_blocks() stub. If 5190 // you want to use it elsewhere, note that cnt must be >= CacheLineSize. 5191 void MacroAssembler::zero_dcache_blocks(Register base, Register cnt, Register tmp1, Register tmp2) { 5192 Label initial_table_end, loop; 5193 5194 // Align base with cache line size. 5195 neg(tmp1, base); 5196 andi(tmp1, tmp1, CacheLineSize - 1); 5197 5198 // tmp1: the number of bytes to be filled to align the base with cache line size. 5199 add(base, base, tmp1); 5200 srai(tmp2, tmp1, 3); 5201 sub(cnt, cnt, tmp2); 5202 srli(tmp2, tmp1, 1); 5203 la(tmp1, initial_table_end); 5204 sub(tmp2, tmp1, tmp2); 5205 jr(tmp2); 5206 for (int i = -CacheLineSize + wordSize; i < 0; i += wordSize) { 5207 sd(zr, Address(base, i)); 5208 } 5209 bind(initial_table_end); 5210 5211 mv(tmp1, CacheLineSize / wordSize); 5212 bind(loop); 5213 cbo_zero(base); 5214 sub(cnt, cnt, tmp1); 5215 add(base, base, CacheLineSize); 5216 bge(cnt, tmp1, loop); 5217 } 5218 5219 // java.lang.Math.round(float a) 5220 // Returns the closest int to the argument, with ties rounding to positive infinity. 5221 void MacroAssembler::java_round_float(Register dst, FloatRegister src, FloatRegister ftmp) { 5222 // this instructions calling sequence provides performance improvement on all tested devices; 5223 // don't change it without re-verification 5224 Label done; 5225 mv(t0, jint_cast(0.5f)); 5226 fmv_w_x(ftmp, t0); 5227 5228 // dst = 0 if NaN 5229 feq_s(t0, src, src); // replacing fclass with feq as performance optimization 5230 mv(dst, zr); 5231 beqz(t0, done); 5232 5233 // dst = (src + 0.5f) rounded down towards negative infinity 5234 // Adding 0.5f to some floats exceeds the precision limits for a float and rounding takes place. 5235 // RDN is required for fadd_s, RNE gives incorrect results: 5236 // -------------------------------------------------------------------- 5237 // fadd.s rne (src + 0.5f): src = 8388609.000000 ftmp = 8388610.000000 5238 // fcvt.w.s rdn: ftmp = 8388610.000000 dst = 8388610 5239 // -------------------------------------------------------------------- 5240 // fadd.s rdn (src + 0.5f): src = 8388609.000000 ftmp = 8388609.000000 5241 // fcvt.w.s rdn: ftmp = 8388609.000000 dst = 8388609 5242 // -------------------------------------------------------------------- 5243 fadd_s(ftmp, src, ftmp, RoundingMode::rdn); 5244 fcvt_w_s(dst, ftmp, RoundingMode::rdn); 5245 5246 bind(done); 5247 } 5248 5249 // java.lang.Math.round(double a) 5250 // Returns the closest long to the argument, with ties rounding to positive infinity. 5251 void MacroAssembler::java_round_double(Register dst, FloatRegister src, FloatRegister ftmp) { 5252 // this instructions calling sequence provides performance improvement on all tested devices; 5253 // don't change it without re-verification 5254 Label done; 5255 mv(t0, julong_cast(0.5)); 5256 fmv_d_x(ftmp, t0); 5257 5258 // dst = 0 if NaN 5259 feq_d(t0, src, src); // replacing fclass with feq as performance optimization 5260 mv(dst, zr); 5261 beqz(t0, done); 5262 5263 // dst = (src + 0.5) rounded down towards negative infinity 5264 fadd_d(ftmp, src, ftmp, RoundingMode::rdn); // RDN is required here otherwise some inputs produce incorrect results 5265 fcvt_l_d(dst, ftmp, RoundingMode::rdn); 5266 5267 bind(done); 5268 } 5269 5270 #define FCVT_SAFE(FLOATCVT, FLOATSIG) \ 5271 void MacroAssembler::FLOATCVT##_safe(Register dst, FloatRegister src, Register tmp) { \ 5272 Label done; \ 5273 assert_different_registers(dst, tmp); \ 5274 fclass_##FLOATSIG(tmp, src); \ 5275 mv(dst, zr); \ 5276 /* check if src is NaN */ \ 5277 andi(tmp, tmp, fclass_mask::nan); \ 5278 bnez(tmp, done); \ 5279 FLOATCVT(dst, src); \ 5280 bind(done); \ 5281 } 5282 5283 FCVT_SAFE(fcvt_w_s, s); 5284 FCVT_SAFE(fcvt_l_s, s); 5285 FCVT_SAFE(fcvt_w_d, d); 5286 FCVT_SAFE(fcvt_l_d, d); 5287 5288 #undef FCVT_SAFE 5289 5290 #define FCMP(FLOATTYPE, FLOATSIG) \ 5291 void MacroAssembler::FLOATTYPE##_compare(Register result, FloatRegister Rs1, \ 5292 FloatRegister Rs2, int unordered_result) { \ 5293 Label Ldone; \ 5294 if (unordered_result < 0) { \ 5295 /* we want -1 for unordered or less than, 0 for equal and 1 for greater than. */ \ 5296 /* installs 1 if gt else 0 */ \ 5297 flt_##FLOATSIG(result, Rs2, Rs1); \ 5298 /* Rs1 > Rs2, install 1 */ \ 5299 bgtz(result, Ldone); \ 5300 feq_##FLOATSIG(result, Rs1, Rs2); \ 5301 addi(result, result, -1); \ 5302 /* Rs1 = Rs2, install 0 */ \ 5303 /* NaN or Rs1 < Rs2, install -1 */ \ 5304 bind(Ldone); \ 5305 } else { \ 5306 /* we want -1 for less than, 0 for equal and 1 for unordered or greater than. */ \ 5307 /* installs 1 if gt or unordered else 0 */ \ 5308 flt_##FLOATSIG(result, Rs1, Rs2); \ 5309 /* Rs1 < Rs2, install -1 */ \ 5310 bgtz(result, Ldone); \ 5311 feq_##FLOATSIG(result, Rs1, Rs2); \ 5312 addi(result, result, -1); \ 5313 /* Rs1 = Rs2, install 0 */ \ 5314 /* NaN or Rs1 > Rs2, install 1 */ \ 5315 bind(Ldone); \ 5316 neg(result, result); \ 5317 } \ 5318 } 5319 5320 FCMP(float, s); 5321 FCMP(double, d); 5322 5323 #undef FCMP 5324 5325 // Zero words; len is in bytes 5326 // Destroys all registers except addr 5327 // len must be a nonzero multiple of wordSize 5328 void MacroAssembler::zero_memory(Register addr, Register len, Register tmp) { 5329 assert_different_registers(addr, len, tmp, t0, t1); 5330 5331 #ifdef ASSERT 5332 { 5333 Label L; 5334 andi(t0, len, BytesPerWord - 1); 5335 beqz(t0, L); 5336 stop("len is not a multiple of BytesPerWord"); 5337 bind(L); 5338 } 5339 #endif // ASSERT 5340 5341 #ifndef PRODUCT 5342 block_comment("zero memory"); 5343 #endif // PRODUCT 5344 5345 Label loop; 5346 Label entry; 5347 5348 // Algorithm: 5349 // 5350 // t0 = cnt & 7 5351 // cnt -= t0 5352 // p += t0 5353 // switch (t0) { 5354 // do { 5355 // cnt -= 8 5356 // p[-8] = 0 5357 // case 7: 5358 // p[-7] = 0 5359 // case 6: 5360 // p[-6] = 0 5361 // ... 5362 // case 1: 5363 // p[-1] = 0 5364 // case 0: 5365 // p += 8 5366 // } while (cnt) 5367 // } 5368 5369 const int unroll = 8; // Number of sd(zr) instructions we'll unroll 5370 5371 srli(len, len, LogBytesPerWord); 5372 andi(t0, len, unroll - 1); // t0 = cnt % unroll 5373 sub(len, len, t0); // cnt -= unroll 5374 // tmp always points to the end of the region we're about to zero 5375 shadd(tmp, t0, addr, t1, LogBytesPerWord); 5376 la(t1, entry); 5377 slli(t0, t0, 2); 5378 sub(t1, t1, t0); 5379 jr(t1); 5380 bind(loop); 5381 sub(len, len, unroll); 5382 for (int i = -unroll; i < 0; i++) { 5383 sd(zr, Address(tmp, i * wordSize)); 5384 } 5385 bind(entry); 5386 add(tmp, tmp, unroll * wordSize); 5387 bnez(len, loop); 5388 } 5389 5390 // shift left by shamt and add 5391 // Rd = (Rs1 << shamt) + Rs2 5392 void MacroAssembler::shadd(Register Rd, Register Rs1, Register Rs2, Register tmp, int shamt) { 5393 if (UseZba) { 5394 if (shamt == 1) { 5395 sh1add(Rd, Rs1, Rs2); 5396 return; 5397 } else if (shamt == 2) { 5398 sh2add(Rd, Rs1, Rs2); 5399 return; 5400 } else if (shamt == 3) { 5401 sh3add(Rd, Rs1, Rs2); 5402 return; 5403 } 5404 } 5405 5406 if (shamt != 0) { 5407 assert_different_registers(Rs2, tmp); 5408 slli(tmp, Rs1, shamt); 5409 add(Rd, Rs2, tmp); 5410 } else { 5411 add(Rd, Rs1, Rs2); 5412 } 5413 } 5414 5415 void MacroAssembler::zero_extend(Register dst, Register src, int bits) { 5416 switch (bits) { 5417 case 32: 5418 if (UseZba) { 5419 zext_w(dst, src); 5420 return; 5421 } 5422 break; 5423 case 16: 5424 if (UseZbb) { 5425 zext_h(dst, src); 5426 return; 5427 } 5428 break; 5429 case 8: 5430 if (UseZbb) { 5431 zext_b(dst, src); 5432 return; 5433 } 5434 break; 5435 default: 5436 break; 5437 } 5438 slli(dst, src, XLEN - bits); 5439 srli(dst, dst, XLEN - bits); 5440 } 5441 5442 void MacroAssembler::sign_extend(Register dst, Register src, int bits) { 5443 switch (bits) { 5444 case 32: 5445 sext_w(dst, src); 5446 return; 5447 case 16: 5448 if (UseZbb) { 5449 sext_h(dst, src); 5450 return; 5451 } 5452 break; 5453 case 8: 5454 if (UseZbb) { 5455 sext_b(dst, src); 5456 return; 5457 } 5458 break; 5459 default: 5460 break; 5461 } 5462 slli(dst, src, XLEN - bits); 5463 srai(dst, dst, XLEN - bits); 5464 } 5465 5466 void MacroAssembler::cmp_x2i(Register dst, Register src1, Register src2, 5467 Register tmp, bool is_signed) { 5468 if (src1 == src2) { 5469 mv(dst, zr); 5470 return; 5471 } 5472 Label done; 5473 Register left = src1; 5474 Register right = src2; 5475 if (dst == src1) { 5476 assert_different_registers(dst, src2, tmp); 5477 mv(tmp, src1); 5478 left = tmp; 5479 } else if (dst == src2) { 5480 assert_different_registers(dst, src1, tmp); 5481 mv(tmp, src2); 5482 right = tmp; 5483 } 5484 5485 // installs 1 if gt else 0 5486 if (is_signed) { 5487 slt(dst, right, left); 5488 } else { 5489 sltu(dst, right, left); 5490 } 5491 bnez(dst, done); 5492 if (is_signed) { 5493 slt(dst, left, right); 5494 } else { 5495 sltu(dst, left, right); 5496 } 5497 // dst = -1 if lt; else if eq , dst = 0 5498 neg(dst, dst); 5499 bind(done); 5500 } 5501 5502 void MacroAssembler::cmp_l2i(Register dst, Register src1, Register src2, Register tmp) 5503 { 5504 cmp_x2i(dst, src1, src2, tmp); 5505 } 5506 5507 void MacroAssembler::cmp_ul2i(Register dst, Register src1, Register src2, Register tmp) { 5508 cmp_x2i(dst, src1, src2, tmp, false); 5509 } 5510 5511 void MacroAssembler::cmp_uw2i(Register dst, Register src1, Register src2, Register tmp) { 5512 cmp_x2i(dst, src1, src2, tmp, false); 5513 } 5514 5515 // The java_calling_convention describes stack locations as ideal slots on 5516 // a frame with no abi restrictions. Since we must observe abi restrictions 5517 // (like the placement of the register window) the slots must be biased by 5518 // the following value. 5519 static int reg2offset_in(VMReg r) { 5520 // Account for saved fp and ra 5521 // This should really be in_preserve_stack_slots 5522 return r->reg2stack() * VMRegImpl::stack_slot_size; 5523 } 5524 5525 static int reg2offset_out(VMReg r) { 5526 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; 5527 } 5528 5529 // The C ABI specifies: 5530 // "integer scalars narrower than XLEN bits are widened according to the sign 5531 // of their type up to 32 bits, then sign-extended to XLEN bits." 5532 // Applies for both passed in register and stack. 5533 // 5534 // Java uses 32-bit stack slots; jint, jshort, jchar, jbyte uses one slot. 5535 // Native uses 64-bit stack slots for all integer scalar types. 5536 // 5537 // lw loads the Java stack slot, sign-extends and 5538 // sd store this widened integer into a 64 bit native stack slot. 5539 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp) { 5540 if (src.first()->is_stack()) { 5541 if (dst.first()->is_stack()) { 5542 // stack to stack 5543 lw(tmp, Address(fp, reg2offset_in(src.first()))); 5544 sd(tmp, Address(sp, reg2offset_out(dst.first()))); 5545 } else { 5546 // stack to reg 5547 lw(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); 5548 } 5549 } else if (dst.first()->is_stack()) { 5550 // reg to stack 5551 sd(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first()))); 5552 } else { 5553 if (dst.first() != src.first()) { 5554 sign_extend(dst.first()->as_Register(), src.first()->as_Register(), 32); 5555 } 5556 } 5557 } 5558 5559 // An oop arg. Must pass a handle not the oop itself 5560 void MacroAssembler::object_move(OopMap* map, 5561 int oop_handle_offset, 5562 int framesize_in_slots, 5563 VMRegPair src, 5564 VMRegPair dst, 5565 bool is_receiver, 5566 int* receiver_offset) { 5567 assert_cond(map != nullptr && receiver_offset != nullptr); 5568 5569 // must pass a handle. First figure out the location we use as a handle 5570 Register rHandle = dst.first()->is_stack() ? t1 : dst.first()->as_Register(); 5571 5572 // See if oop is null if it is we need no handle 5573 5574 if (src.first()->is_stack()) { 5575 // Oop is already on the stack as an argument 5576 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots(); 5577 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots)); 5578 if (is_receiver) { 5579 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size; 5580 } 5581 5582 ld(t0, Address(fp, reg2offset_in(src.first()))); 5583 la(rHandle, Address(fp, reg2offset_in(src.first()))); 5584 // conditionally move a null 5585 Label notZero1; 5586 bnez(t0, notZero1); 5587 mv(rHandle, zr); 5588 bind(notZero1); 5589 } else { 5590 5591 // Oop is in a register we must store it to the space we reserve 5592 // on the stack for oop_handles and pass a handle if oop is non-null 5593 5594 const Register rOop = src.first()->as_Register(); 5595 int oop_slot = -1; 5596 if (rOop == j_rarg0) { 5597 oop_slot = 0; 5598 } else if (rOop == j_rarg1) { 5599 oop_slot = 1; 5600 } else if (rOop == j_rarg2) { 5601 oop_slot = 2; 5602 } else if (rOop == j_rarg3) { 5603 oop_slot = 3; 5604 } else if (rOop == j_rarg4) { 5605 oop_slot = 4; 5606 } else if (rOop == j_rarg5) { 5607 oop_slot = 5; 5608 } else if (rOop == j_rarg6) { 5609 oop_slot = 6; 5610 } else { 5611 assert(rOop == j_rarg7, "wrong register"); 5612 oop_slot = 7; 5613 } 5614 5615 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset; 5616 int offset = oop_slot * VMRegImpl::stack_slot_size; 5617 5618 map->set_oop(VMRegImpl::stack2reg(oop_slot)); 5619 // Store oop in handle area, may be null 5620 sd(rOop, Address(sp, offset)); 5621 if (is_receiver) { 5622 *receiver_offset = offset; 5623 } 5624 5625 //rOop maybe the same as rHandle 5626 if (rOop == rHandle) { 5627 Label isZero; 5628 beqz(rOop, isZero); 5629 la(rHandle, Address(sp, offset)); 5630 bind(isZero); 5631 } else { 5632 Label notZero2; 5633 la(rHandle, Address(sp, offset)); 5634 bnez(rOop, notZero2); 5635 mv(rHandle, zr); 5636 bind(notZero2); 5637 } 5638 } 5639 5640 // If arg is on the stack then place it otherwise it is already in correct reg. 5641 if (dst.first()->is_stack()) { 5642 sd(rHandle, Address(sp, reg2offset_out(dst.first()))); 5643 } 5644 } 5645 5646 // A float arg may have to do float reg int reg conversion 5647 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp) { 5648 assert((src.first()->is_stack() && dst.first()->is_stack()) || 5649 (src.first()->is_reg() && dst.first()->is_reg()) || 5650 (src.first()->is_stack() && dst.first()->is_reg()), "Unexpected error"); 5651 if (src.first()->is_stack()) { 5652 if (dst.first()->is_stack()) { 5653 lwu(tmp, Address(fp, reg2offset_in(src.first()))); 5654 sw(tmp, Address(sp, reg2offset_out(dst.first()))); 5655 } else if (dst.first()->is_Register()) { 5656 lwu(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); 5657 } else { 5658 ShouldNotReachHere(); 5659 } 5660 } else if (src.first() != dst.first()) { 5661 if (src.is_single_phys_reg() && dst.is_single_phys_reg()) { 5662 fmv_s(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister()); 5663 } else { 5664 ShouldNotReachHere(); 5665 } 5666 } 5667 } 5668 5669 // A long move 5670 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp) { 5671 if (src.first()->is_stack()) { 5672 if (dst.first()->is_stack()) { 5673 // stack to stack 5674 ld(tmp, Address(fp, reg2offset_in(src.first()))); 5675 sd(tmp, Address(sp, reg2offset_out(dst.first()))); 5676 } else { 5677 // stack to reg 5678 ld(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); 5679 } 5680 } else if (dst.first()->is_stack()) { 5681 // reg to stack 5682 sd(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first()))); 5683 } else { 5684 if (dst.first() != src.first()) { 5685 mv(dst.first()->as_Register(), src.first()->as_Register()); 5686 } 5687 } 5688 } 5689 5690 // A double move 5691 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) { 5692 assert((src.first()->is_stack() && dst.first()->is_stack()) || 5693 (src.first()->is_reg() && dst.first()->is_reg()) || 5694 (src.first()->is_stack() && dst.first()->is_reg()), "Unexpected error"); 5695 if (src.first()->is_stack()) { 5696 if (dst.first()->is_stack()) { 5697 ld(tmp, Address(fp, reg2offset_in(src.first()))); 5698 sd(tmp, Address(sp, reg2offset_out(dst.first()))); 5699 } else if (dst.first()-> is_Register()) { 5700 ld(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); 5701 } else { 5702 ShouldNotReachHere(); 5703 } 5704 } else if (src.first() != dst.first()) { 5705 if (src.is_single_phys_reg() && dst.is_single_phys_reg()) { 5706 fmv_d(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister()); 5707 } else { 5708 ShouldNotReachHere(); 5709 } 5710 } 5711 } 5712 5713 void MacroAssembler::test_bit(Register Rd, Register Rs, uint32_t bit_pos) { 5714 assert(bit_pos < 64, "invalid bit range"); 5715 if (UseZbs) { 5716 bexti(Rd, Rs, bit_pos); 5717 return; 5718 } 5719 int64_t imm = (int64_t)(1UL << bit_pos); 5720 if (is_simm12(imm)) { 5721 and_imm12(Rd, Rs, imm); 5722 } else { 5723 srli(Rd, Rs, bit_pos); 5724 and_imm12(Rd, Rd, 1); 5725 } 5726 } 5727 5728 // Implements lightweight-locking. 5729 // 5730 // - obj: the object to be locked 5731 // - tmp1, tmp2, tmp3: temporary registers, will be destroyed 5732 // - slow: branched to if locking fails 5733 void MacroAssembler::lightweight_lock(Register basic_lock, Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow) { 5734 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking"); 5735 assert_different_registers(basic_lock, obj, tmp1, tmp2, tmp3, t0); 5736 5737 Label push; 5738 const Register top = tmp1; 5739 const Register mark = tmp2; 5740 const Register t = tmp3; 5741 5742 // Preload the markWord. It is important that this is the first 5743 // instruction emitted as it is part of C1's null check semantics. 5744 ld(mark, Address(obj, oopDesc::mark_offset_in_bytes())); 5745 5746 if (UseObjectMonitorTable) { 5747 // Clear cache in case fast locking succeeds. 5748 sd(zr, Address(basic_lock, BasicObjectLock::lock_offset() + in_ByteSize((BasicLock::object_monitor_cache_offset_in_bytes())))); 5749 } 5750 5751 // Check if the lock-stack is full. 5752 lwu(top, Address(xthread, JavaThread::lock_stack_top_offset())); 5753 mv(t, (unsigned)LockStack::end_offset()); 5754 bge(top, t, slow, /* is_far */ true); 5755 5756 // Check for recursion. 5757 add(t, xthread, top); 5758 ld(t, Address(t, -oopSize)); 5759 beq(obj, t, push); 5760 5761 // Check header for monitor (0b10). 5762 test_bit(t, mark, exact_log2(markWord::monitor_value)); 5763 bnez(t, slow, /* is_far */ true); 5764 5765 // Try to lock. Transition lock-bits 0b01 => 0b00 5766 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid a la"); 5767 ori(mark, mark, markWord::unlocked_value); 5768 xori(t, mark, markWord::unlocked_value); 5769 cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64, 5770 /*acquire*/ Assembler::aq, /*release*/ Assembler::relaxed, /*result*/ t); 5771 bne(mark, t, slow, /* is_far */ true); 5772 5773 bind(push); 5774 // After successful lock, push object on lock-stack. 5775 add(t, xthread, top); 5776 sd(obj, Address(t)); 5777 addw(top, top, oopSize); 5778 sw(top, Address(xthread, JavaThread::lock_stack_top_offset())); 5779 } 5780 5781 // Implements ligthweight-unlocking. 5782 // 5783 // - obj: the object to be unlocked 5784 // - tmp1, tmp2, tmp3: temporary registers 5785 // - slow: branched to if unlocking fails 5786 void MacroAssembler::lightweight_unlock(Register obj, Register tmp1, Register tmp2, Register tmp3, Label& slow) { 5787 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking"); 5788 assert_different_registers(obj, tmp1, tmp2, tmp3, t0); 5789 5790 #ifdef ASSERT 5791 { 5792 // Check for lock-stack underflow. 5793 Label stack_ok; 5794 lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset())); 5795 mv(tmp2, (unsigned)LockStack::start_offset()); 5796 bge(tmp1, tmp2, stack_ok); 5797 STOP("Lock-stack underflow"); 5798 bind(stack_ok); 5799 } 5800 #endif 5801 5802 Label unlocked, push_and_slow; 5803 const Register top = tmp1; 5804 const Register mark = tmp2; 5805 const Register t = tmp3; 5806 5807 // Check if obj is top of lock-stack. 5808 lwu(top, Address(xthread, JavaThread::lock_stack_top_offset())); 5809 subw(top, top, oopSize); 5810 add(t, xthread, top); 5811 ld(t, Address(t)); 5812 bne(obj, t, slow, /* is_far */ true); 5813 5814 // Pop lock-stack. 5815 DEBUG_ONLY(add(t, xthread, top);) 5816 DEBUG_ONLY(sd(zr, Address(t));) 5817 sw(top, Address(xthread, JavaThread::lock_stack_top_offset())); 5818 5819 // Check if recursive. 5820 add(t, xthread, top); 5821 ld(t, Address(t, -oopSize)); 5822 beq(obj, t, unlocked); 5823 5824 // Not recursive. Check header for monitor (0b10). 5825 ld(mark, Address(obj, oopDesc::mark_offset_in_bytes())); 5826 test_bit(t, mark, exact_log2(markWord::monitor_value)); 5827 bnez(t, push_and_slow); 5828 5829 #ifdef ASSERT 5830 // Check header not unlocked (0b01). 5831 Label not_unlocked; 5832 test_bit(t, mark, exact_log2(markWord::unlocked_value)); 5833 beqz(t, not_unlocked); 5834 stop("lightweight_unlock already unlocked"); 5835 bind(not_unlocked); 5836 #endif 5837 5838 // Try to unlock. Transition lock bits 0b00 => 0b01 5839 assert(oopDesc::mark_offset_in_bytes() == 0, "required to avoid lea"); 5840 ori(t, mark, markWord::unlocked_value); 5841 cmpxchg(/*addr*/ obj, /*expected*/ mark, /*new*/ t, Assembler::int64, 5842 /*acquire*/ Assembler::relaxed, /*release*/ Assembler::rl, /*result*/ t); 5843 beq(mark, t, unlocked); 5844 5845 bind(push_and_slow); 5846 // Restore lock-stack and handle the unlock in runtime. 5847 DEBUG_ONLY(add(t, xthread, top);) 5848 DEBUG_ONLY(sd(obj, Address(t));) 5849 addw(top, top, oopSize); 5850 sw(top, Address(xthread, JavaThread::lock_stack_top_offset())); 5851 j(slow); 5852 5853 bind(unlocked); 5854 }