1 /* 2 * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved. 3 * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved. 4 * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved. 5 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 * 7 * This code is free software; you can redistribute it and/or modify it 8 * under the terms of the GNU General Public License version 2 only, as 9 * published by the Free Software Foundation. 10 * 11 * This code is distributed in the hope that it will be useful, but WITHOUT 12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or 13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14 * version 2 for more details (a copy is included in the LICENSE file that 15 * accompanied this code). 16 * 17 * You should have received a copy of the GNU General Public License version 18 * 2 along with this work; if not, write to the Free Software Foundation, 19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. 20 * 21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA 22 * or visit www.oracle.com if you need additional information or have any 23 * questions. 24 * 25 */ 26 27 #include "precompiled.hpp" 28 #include "asm/assembler.hpp" 29 #include "asm/assembler.inline.hpp" 30 #include "compiler/disassembler.hpp" 31 #include "gc/shared/barrierSet.hpp" 32 #include "gc/shared/barrierSetAssembler.hpp" 33 #include "gc/shared/cardTable.hpp" 34 #include "gc/shared/cardTableBarrierSet.hpp" 35 #include "gc/shared/collectedHeap.hpp" 36 #include "interpreter/bytecodeHistogram.hpp" 37 #include "interpreter/interpreter.hpp" 38 #include "memory/resourceArea.hpp" 39 #include "memory/universe.hpp" 40 #include "nativeInst_riscv.hpp" 41 #include "oops/accessDecorators.hpp" 42 #include "oops/compressedOops.inline.hpp" 43 #include "oops/klass.inline.hpp" 44 #include "oops/oop.hpp" 45 #include "runtime/interfaceSupport.inline.hpp" 46 #include "runtime/javaThread.hpp" 47 #include "runtime/jniHandles.inline.hpp" 48 #include "runtime/sharedRuntime.hpp" 49 #include "runtime/stubRoutines.hpp" 50 #include "utilities/powerOfTwo.hpp" 51 #ifdef COMPILER2 52 #include "opto/compile.hpp" 53 #include "opto/node.hpp" 54 #include "opto/output.hpp" 55 #endif 56 57 #ifdef PRODUCT 58 #define BLOCK_COMMENT(str) /* nothing */ 59 #else 60 #define BLOCK_COMMENT(str) block_comment(str) 61 #endif 62 #define STOP(str) stop(str); 63 #define BIND(label) bind(label); __ BLOCK_COMMENT(#label ":") 64 65 static void pass_arg0(MacroAssembler* masm, Register arg) { 66 if (c_rarg0 != arg) { 67 masm->mv(c_rarg0, arg); 68 } 69 } 70 71 static void pass_arg1(MacroAssembler* masm, Register arg) { 72 if (c_rarg1 != arg) { 73 masm->mv(c_rarg1, arg); 74 } 75 } 76 77 static void pass_arg2(MacroAssembler* masm, Register arg) { 78 if (c_rarg2 != arg) { 79 masm->mv(c_rarg2, arg); 80 } 81 } 82 83 static void pass_arg3(MacroAssembler* masm, Register arg) { 84 if (c_rarg3 != arg) { 85 masm->mv(c_rarg3, arg); 86 } 87 } 88 89 void MacroAssembler::push_cont_fastpath(Register java_thread) { 90 if (!Continuations::enabled()) return; 91 Label done; 92 ld(t0, Address(java_thread, JavaThread::cont_fastpath_offset())); 93 bleu(sp, t0, done); 94 sd(sp, Address(java_thread, JavaThread::cont_fastpath_offset())); 95 bind(done); 96 } 97 98 void MacroAssembler::pop_cont_fastpath(Register java_thread) { 99 if (!Continuations::enabled()) return; 100 Label done; 101 ld(t0, Address(java_thread, JavaThread::cont_fastpath_offset())); 102 bltu(sp, t0, done); 103 sd(zr, Address(java_thread, JavaThread::cont_fastpath_offset())); 104 bind(done); 105 } 106 107 int MacroAssembler::align(int modulus, int extra_offset) { 108 CompressibleRegion cr(this); 109 intptr_t before = offset(); 110 while ((offset() + extra_offset) % modulus != 0) { nop(); } 111 return (int)(offset() - before); 112 } 113 114 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) { 115 call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions); 116 } 117 118 // Implementation of call_VM versions 119 120 void MacroAssembler::call_VM(Register oop_result, 121 address entry_point, 122 bool check_exceptions) { 123 call_VM_helper(oop_result, entry_point, 0, check_exceptions); 124 } 125 126 void MacroAssembler::call_VM(Register oop_result, 127 address entry_point, 128 Register arg_1, 129 bool check_exceptions) { 130 pass_arg1(this, arg_1); 131 call_VM_helper(oop_result, entry_point, 1, check_exceptions); 132 } 133 134 void MacroAssembler::call_VM(Register oop_result, 135 address entry_point, 136 Register arg_1, 137 Register arg_2, 138 bool check_exceptions) { 139 assert(arg_1 != c_rarg2, "smashed arg"); 140 pass_arg2(this, arg_2); 141 pass_arg1(this, arg_1); 142 call_VM_helper(oop_result, entry_point, 2, check_exceptions); 143 } 144 145 void MacroAssembler::call_VM(Register oop_result, 146 address entry_point, 147 Register arg_1, 148 Register arg_2, 149 Register arg_3, 150 bool check_exceptions) { 151 assert(arg_1 != c_rarg3, "smashed arg"); 152 assert(arg_2 != c_rarg3, "smashed arg"); 153 pass_arg3(this, arg_3); 154 155 assert(arg_1 != c_rarg2, "smashed arg"); 156 pass_arg2(this, arg_2); 157 158 pass_arg1(this, arg_1); 159 call_VM_helper(oop_result, entry_point, 3, check_exceptions); 160 } 161 162 void MacroAssembler::call_VM(Register oop_result, 163 Register last_java_sp, 164 address entry_point, 165 int number_of_arguments, 166 bool check_exceptions) { 167 call_VM_base(oop_result, xthread, last_java_sp, entry_point, number_of_arguments, check_exceptions); 168 } 169 170 void MacroAssembler::call_VM(Register oop_result, 171 Register last_java_sp, 172 address entry_point, 173 Register arg_1, 174 bool check_exceptions) { 175 pass_arg1(this, arg_1); 176 call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions); 177 } 178 179 void MacroAssembler::call_VM(Register oop_result, 180 Register last_java_sp, 181 address entry_point, 182 Register arg_1, 183 Register arg_2, 184 bool check_exceptions) { 185 186 assert(arg_1 != c_rarg2, "smashed arg"); 187 pass_arg2(this, arg_2); 188 pass_arg1(this, arg_1); 189 call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions); 190 } 191 192 void MacroAssembler::call_VM(Register oop_result, 193 Register last_java_sp, 194 address entry_point, 195 Register arg_1, 196 Register arg_2, 197 Register arg_3, 198 bool check_exceptions) { 199 assert(arg_1 != c_rarg3, "smashed arg"); 200 assert(arg_2 != c_rarg3, "smashed arg"); 201 pass_arg3(this, arg_3); 202 assert(arg_1 != c_rarg2, "smashed arg"); 203 pass_arg2(this, arg_2); 204 pass_arg1(this, arg_1); 205 call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions); 206 } 207 208 void MacroAssembler::post_call_nop() { 209 if (!Continuations::enabled()) { 210 return; 211 } 212 relocate(post_call_nop_Relocation::spec(), [&] { 213 InlineSkippedInstructionsCounter skipCounter(this); 214 nop(); 215 li32(zr, 0); 216 }); 217 } 218 219 // these are no-ops overridden by InterpreterMacroAssembler 220 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {} 221 void MacroAssembler::check_and_handle_popframe(Register java_thread) {} 222 223 // Calls to C land 224 // 225 // When entering C land, the fp, & esp of the last Java frame have to be recorded 226 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp 227 // has to be reset to 0. This is required to allow proper stack traversal. 228 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 229 Register last_java_fp, 230 Register last_java_pc, 231 Register tmp) { 232 233 if (last_java_pc->is_valid()) { 234 sd(last_java_pc, Address(xthread, 235 JavaThread::frame_anchor_offset() + 236 JavaFrameAnchor::last_Java_pc_offset())); 237 } 238 239 // determine last_java_sp register 240 if (last_java_sp == sp) { 241 mv(tmp, sp); 242 last_java_sp = tmp; 243 } else if (!last_java_sp->is_valid()) { 244 last_java_sp = esp; 245 } 246 247 sd(last_java_sp, Address(xthread, JavaThread::last_Java_sp_offset())); 248 249 // last_java_fp is optional 250 if (last_java_fp->is_valid()) { 251 sd(last_java_fp, Address(xthread, JavaThread::last_Java_fp_offset())); 252 } 253 } 254 255 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 256 Register last_java_fp, 257 address last_java_pc, 258 Register tmp) { 259 assert(last_java_pc != nullptr, "must provide a valid PC"); 260 261 la(tmp, last_java_pc); 262 sd(tmp, Address(xthread, JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset())); 263 264 set_last_Java_frame(last_java_sp, last_java_fp, noreg, tmp); 265 } 266 267 void MacroAssembler::set_last_Java_frame(Register last_java_sp, 268 Register last_java_fp, 269 Label &L, 270 Register tmp) { 271 if (L.is_bound()) { 272 set_last_Java_frame(last_java_sp, last_java_fp, target(L), tmp); 273 } else { 274 L.add_patch_at(code(), locator()); 275 IncompressibleRegion ir(this); // the label address will be patched back. 276 set_last_Java_frame(last_java_sp, last_java_fp, pc() /* Patched later */, tmp); 277 } 278 } 279 280 void MacroAssembler::reset_last_Java_frame(bool clear_fp) { 281 // we must set sp to zero to clear frame 282 sd(zr, Address(xthread, JavaThread::last_Java_sp_offset())); 283 284 // must clear fp, so that compiled frames are not confused; it is 285 // possible that we need it only for debugging 286 if (clear_fp) { 287 sd(zr, Address(xthread, JavaThread::last_Java_fp_offset())); 288 } 289 290 // Always clear the pc because it could have been set by make_walkable() 291 sd(zr, Address(xthread, JavaThread::last_Java_pc_offset())); 292 } 293 294 void MacroAssembler::call_VM_base(Register oop_result, 295 Register java_thread, 296 Register last_java_sp, 297 address entry_point, 298 int number_of_arguments, 299 bool check_exceptions) { 300 // determine java_thread register 301 if (!java_thread->is_valid()) { 302 java_thread = xthread; 303 } 304 // determine last_java_sp register 305 if (!last_java_sp->is_valid()) { 306 last_java_sp = esp; 307 } 308 309 // debugging support 310 assert(number_of_arguments >= 0 , "cannot have negative number of arguments"); 311 assert(java_thread == xthread, "unexpected register"); 312 313 assert(java_thread != oop_result , "cannot use the same register for java_thread & oop_result"); 314 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp"); 315 316 // push java thread (becomes first argument of C function) 317 mv(c_rarg0, java_thread); 318 319 // set last Java frame before call 320 assert(last_java_sp != fp, "can't use fp"); 321 322 Label l; 323 set_last_Java_frame(last_java_sp, fp, l, t0); 324 325 // do the call, remove parameters 326 MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l); 327 328 // reset last Java frame 329 // Only interpreter should have to clear fp 330 reset_last_Java_frame(true); 331 332 // C++ interp handles this in the interpreter 333 check_and_handle_popframe(java_thread); 334 check_and_handle_earlyret(java_thread); 335 336 if (check_exceptions) { 337 // check for pending exceptions (java_thread is set upon return) 338 ld(t0, Address(java_thread, in_bytes(Thread::pending_exception_offset()))); 339 Label ok; 340 beqz(t0, ok); 341 RuntimeAddress target(StubRoutines::forward_exception_entry()); 342 relocate(target.rspec(), [&] { 343 int32_t offset; 344 la_patchable(t0, target, offset); 345 jalr(x0, t0, offset); 346 }); 347 bind(ok); 348 } 349 350 // get oop result if there is one and reset the value in the thread 351 if (oop_result->is_valid()) { 352 get_vm_result(oop_result, java_thread); 353 } 354 } 355 356 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) { 357 ld(oop_result, Address(java_thread, JavaThread::vm_result_offset())); 358 sd(zr, Address(java_thread, JavaThread::vm_result_offset())); 359 verify_oop_msg(oop_result, "broken oop in call_VM_base"); 360 } 361 362 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) { 363 ld(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset())); 364 sd(zr, Address(java_thread, JavaThread::vm_result_2_offset())); 365 } 366 367 void MacroAssembler::clinit_barrier(Register klass, Register tmp, Label* L_fast_path, Label* L_slow_path) { 368 assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required"); 369 assert_different_registers(klass, xthread, tmp); 370 371 Label L_fallthrough, L_tmp; 372 if (L_fast_path == nullptr) { 373 L_fast_path = &L_fallthrough; 374 } else if (L_slow_path == nullptr) { 375 L_slow_path = &L_fallthrough; 376 } 377 378 // Fast path check: class is fully initialized 379 lbu(tmp, Address(klass, InstanceKlass::init_state_offset())); 380 sub(tmp, tmp, InstanceKlass::fully_initialized); 381 beqz(tmp, *L_fast_path); 382 383 // Fast path check: current thread is initializer thread 384 ld(tmp, Address(klass, InstanceKlass::init_thread_offset())); 385 386 if (L_slow_path == &L_fallthrough) { 387 beq(xthread, tmp, *L_fast_path); 388 bind(*L_slow_path); 389 } else if (L_fast_path == &L_fallthrough) { 390 bne(xthread, tmp, *L_slow_path); 391 bind(*L_fast_path); 392 } else { 393 Unimplemented(); 394 } 395 } 396 397 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) { 398 if (!VerifyOops) { return; } 399 400 // Pass register number to verify_oop_subroutine 401 const char* b = nullptr; 402 { 403 ResourceMark rm; 404 stringStream ss; 405 ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line); 406 b = code_string(ss.as_string()); 407 } 408 BLOCK_COMMENT("verify_oop {"); 409 410 push_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); 411 412 mv(c_rarg0, reg); // c_rarg0 : x10 413 { 414 // The length of the instruction sequence emitted should not depend 415 // on the address of the char buffer so that the size of mach nodes for 416 // scratch emit and normal emit matches. 417 IncompressibleRegion ir(this); // Fixed length 418 movptr(t0, (address) b); 419 } 420 421 // call indirectly to solve generation ordering problem 422 ExternalAddress target(StubRoutines::verify_oop_subroutine_entry_address()); 423 relocate(target.rspec(), [&] { 424 int32_t offset; 425 la_patchable(t1, target, offset); 426 ld(t1, Address(t1, offset)); 427 }); 428 jalr(t1); 429 430 pop_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); 431 432 BLOCK_COMMENT("} verify_oop"); 433 } 434 435 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) { 436 if (!VerifyOops) { 437 return; 438 } 439 440 const char* b = nullptr; 441 { 442 ResourceMark rm; 443 stringStream ss; 444 ss.print("verify_oop_addr: %s (%s:%d)", s, file, line); 445 b = code_string(ss.as_string()); 446 } 447 BLOCK_COMMENT("verify_oop_addr {"); 448 449 push_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); 450 451 if (addr.uses(sp)) { 452 la(x10, addr); 453 ld(x10, Address(x10, 4 * wordSize)); 454 } else { 455 ld(x10, addr); 456 } 457 458 { 459 // The length of the instruction sequence emitted should not depend 460 // on the address of the char buffer so that the size of mach nodes for 461 // scratch emit and normal emit matches. 462 IncompressibleRegion ir(this); // Fixed length 463 movptr(t0, (address) b); 464 } 465 466 // call indirectly to solve generation ordering problem 467 ExternalAddress target(StubRoutines::verify_oop_subroutine_entry_address()); 468 relocate(target.rspec(), [&] { 469 int32_t offset; 470 la_patchable(t1, target, offset); 471 ld(t1, Address(t1, offset)); 472 }); 473 jalr(t1); 474 475 pop_reg(RegSet::of(ra, t0, t1, c_rarg0), sp); 476 477 BLOCK_COMMENT("} verify_oop_addr"); 478 } 479 480 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot, 481 int extra_slot_offset) { 482 // cf. TemplateTable::prepare_invoke(), if (load_receiver). 483 int stackElementSize = Interpreter::stackElementSize; 484 int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0); 485 #ifdef ASSERT 486 int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1); 487 assert(offset1 - offset == stackElementSize, "correct arithmetic"); 488 #endif 489 if (arg_slot.is_constant()) { 490 return Address(esp, arg_slot.as_constant() * stackElementSize + offset); 491 } else { 492 assert_different_registers(t0, arg_slot.as_register()); 493 shadd(t0, arg_slot.as_register(), esp, t0, exact_log2(stackElementSize)); 494 return Address(t0, offset); 495 } 496 } 497 498 #ifndef PRODUCT 499 extern "C" void findpc(intptr_t x); 500 #endif 501 502 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[]) 503 { 504 // In order to get locks to work, we need to fake a in_VM state 505 if (ShowMessageBoxOnError) { 506 JavaThread* thread = JavaThread::current(); 507 JavaThreadState saved_state = thread->thread_state(); 508 thread->set_thread_state(_thread_in_vm); 509 #ifndef PRODUCT 510 if (CountBytecodes || TraceBytecodes || StopInterpreterAt) { 511 ttyLocker ttyl; 512 BytecodeCounter::print(); 513 } 514 #endif 515 if (os::message_box(msg, "Execution stopped, print registers?")) { 516 ttyLocker ttyl; 517 tty->print_cr(" pc = 0x%016lx", pc); 518 #ifndef PRODUCT 519 tty->cr(); 520 findpc(pc); 521 tty->cr(); 522 #endif 523 tty->print_cr(" x0 = 0x%016lx", regs[0]); 524 tty->print_cr(" x1 = 0x%016lx", regs[1]); 525 tty->print_cr(" x2 = 0x%016lx", regs[2]); 526 tty->print_cr(" x3 = 0x%016lx", regs[3]); 527 tty->print_cr(" x4 = 0x%016lx", regs[4]); 528 tty->print_cr(" x5 = 0x%016lx", regs[5]); 529 tty->print_cr(" x6 = 0x%016lx", regs[6]); 530 tty->print_cr(" x7 = 0x%016lx", regs[7]); 531 tty->print_cr(" x8 = 0x%016lx", regs[8]); 532 tty->print_cr(" x9 = 0x%016lx", regs[9]); 533 tty->print_cr("x10 = 0x%016lx", regs[10]); 534 tty->print_cr("x11 = 0x%016lx", regs[11]); 535 tty->print_cr("x12 = 0x%016lx", regs[12]); 536 tty->print_cr("x13 = 0x%016lx", regs[13]); 537 tty->print_cr("x14 = 0x%016lx", regs[14]); 538 tty->print_cr("x15 = 0x%016lx", regs[15]); 539 tty->print_cr("x16 = 0x%016lx", regs[16]); 540 tty->print_cr("x17 = 0x%016lx", regs[17]); 541 tty->print_cr("x18 = 0x%016lx", regs[18]); 542 tty->print_cr("x19 = 0x%016lx", regs[19]); 543 tty->print_cr("x20 = 0x%016lx", regs[20]); 544 tty->print_cr("x21 = 0x%016lx", regs[21]); 545 tty->print_cr("x22 = 0x%016lx", regs[22]); 546 tty->print_cr("x23 = 0x%016lx", regs[23]); 547 tty->print_cr("x24 = 0x%016lx", regs[24]); 548 tty->print_cr("x25 = 0x%016lx", regs[25]); 549 tty->print_cr("x26 = 0x%016lx", regs[26]); 550 tty->print_cr("x27 = 0x%016lx", regs[27]); 551 tty->print_cr("x28 = 0x%016lx", regs[28]); 552 tty->print_cr("x30 = 0x%016lx", regs[30]); 553 tty->print_cr("x31 = 0x%016lx", regs[31]); 554 BREAKPOINT; 555 } 556 } 557 fatal("DEBUG MESSAGE: %s", msg); 558 } 559 560 void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp2) { 561 assert_different_registers(value, tmp1, tmp2); 562 Label done, tagged, weak_tagged; 563 564 beqz(value, done); // Use null as-is. 565 // Test for tag. 566 andi(tmp1, value, JNIHandles::tag_mask); 567 bnez(tmp1, tagged); 568 569 // Resolve local handle 570 access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp1, tmp2); 571 verify_oop(value); 572 j(done); 573 574 bind(tagged); 575 // Test for jweak tag. 576 STATIC_ASSERT(JNIHandles::TypeTag::weak_global == 0b1); 577 test_bit(tmp1, value, exact_log2(JNIHandles::TypeTag::weak_global)); 578 bnez(tmp1, weak_tagged); 579 580 // Resolve global handle 581 access_load_at(T_OBJECT, IN_NATIVE, value, 582 Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2); 583 verify_oop(value); 584 j(done); 585 586 bind(weak_tagged); 587 // Resolve jweak. 588 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, value, 589 Address(value, -JNIHandles::TypeTag::weak_global), tmp1, tmp2); 590 verify_oop(value); 591 592 bind(done); 593 } 594 595 void MacroAssembler::resolve_global_jobject(Register value, Register tmp1, Register tmp2) { 596 assert_different_registers(value, tmp1, tmp2); 597 Label done; 598 599 beqz(value, done); // Use null as-is. 600 601 #ifdef ASSERT 602 { 603 STATIC_ASSERT(JNIHandles::TypeTag::global == 0b10); 604 Label valid_global_tag; 605 test_bit(tmp1, value, exact_log2(JNIHandles::TypeTag::global)); // Test for global tag. 606 bnez(tmp1, valid_global_tag); 607 stop("non global jobject using resolve_global_jobject"); 608 bind(valid_global_tag); 609 } 610 #endif 611 612 // Resolve global handle 613 access_load_at(T_OBJECT, IN_NATIVE, value, 614 Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2); 615 verify_oop(value); 616 617 bind(done); 618 } 619 620 void MacroAssembler::stop(const char* msg) { 621 BLOCK_COMMENT(msg); 622 illegal_instruction(Assembler::csr::time); 623 emit_int64((uintptr_t)msg); 624 } 625 626 void MacroAssembler::unimplemented(const char* what) { 627 const char* buf = nullptr; 628 { 629 ResourceMark rm; 630 stringStream ss; 631 ss.print("unimplemented: %s", what); 632 buf = code_string(ss.as_string()); 633 } 634 stop(buf); 635 } 636 637 void MacroAssembler::emit_static_call_stub() { 638 IncompressibleRegion ir(this); // Fixed length: see CompiledStaticCall::to_interp_stub_size(). 639 // CompiledDirectStaticCall::set_to_interpreted knows the 640 // exact layout of this stub. 641 642 mov_metadata(xmethod, (Metadata*)nullptr); 643 644 // Jump to the entry point of the c2i stub. 645 int32_t offset = 0; 646 movptr(t0, 0, offset); 647 jalr(x0, t0, offset); 648 } 649 650 void MacroAssembler::call_VM_leaf_base(address entry_point, 651 int number_of_arguments, 652 Label *retaddr) { 653 push_reg(RegSet::of(t0, xmethod), sp); // push << t0 & xmethod >> to sp 654 call(entry_point); 655 if (retaddr != nullptr) { 656 bind(*retaddr); 657 } 658 pop_reg(RegSet::of(t0, xmethod), sp); // pop << t0 & xmethod >> from sp 659 } 660 661 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) { 662 call_VM_leaf_base(entry_point, number_of_arguments); 663 } 664 665 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) { 666 pass_arg0(this, arg_0); 667 call_VM_leaf_base(entry_point, 1); 668 } 669 670 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 671 pass_arg0(this, arg_0); 672 pass_arg1(this, arg_1); 673 call_VM_leaf_base(entry_point, 2); 674 } 675 676 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, 677 Register arg_1, Register arg_2) { 678 pass_arg0(this, arg_0); 679 pass_arg1(this, arg_1); 680 pass_arg2(this, arg_2); 681 call_VM_leaf_base(entry_point, 3); 682 } 683 684 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) { 685 pass_arg0(this, arg_0); 686 MacroAssembler::call_VM_leaf_base(entry_point, 1); 687 } 688 689 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) { 690 691 assert(arg_0 != c_rarg1, "smashed arg"); 692 pass_arg1(this, arg_1); 693 pass_arg0(this, arg_0); 694 MacroAssembler::call_VM_leaf_base(entry_point, 2); 695 } 696 697 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) { 698 assert(arg_0 != c_rarg2, "smashed arg"); 699 assert(arg_1 != c_rarg2, "smashed arg"); 700 pass_arg2(this, arg_2); 701 assert(arg_0 != c_rarg1, "smashed arg"); 702 pass_arg1(this, arg_1); 703 pass_arg0(this, arg_0); 704 MacroAssembler::call_VM_leaf_base(entry_point, 3); 705 } 706 707 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) { 708 assert(arg_0 != c_rarg3, "smashed arg"); 709 assert(arg_1 != c_rarg3, "smashed arg"); 710 assert(arg_2 != c_rarg3, "smashed arg"); 711 pass_arg3(this, arg_3); 712 assert(arg_0 != c_rarg2, "smashed arg"); 713 assert(arg_1 != c_rarg2, "smashed arg"); 714 pass_arg2(this, arg_2); 715 assert(arg_0 != c_rarg1, "smashed arg"); 716 pass_arg1(this, arg_1); 717 pass_arg0(this, arg_0); 718 MacroAssembler::call_VM_leaf_base(entry_point, 4); 719 } 720 721 void MacroAssembler::la(Register Rd, const address dest) { 722 int64_t offset = dest - pc(); 723 if (is_simm32(offset)) { 724 auipc(Rd, (int32_t)offset + 0x800); //0x800, Note:the 11th sign bit 725 addi(Rd, Rd, ((int64_t)offset << 52) >> 52); 726 } else { 727 movptr(Rd, dest); 728 } 729 } 730 731 void MacroAssembler::la(Register Rd, const Address &adr) { 732 switch (adr.getMode()) { 733 case Address::literal: { 734 relocInfo::relocType rtype = adr.rspec().reloc()->type(); 735 if (rtype == relocInfo::none) { 736 mv(Rd, (intptr_t)(adr.target())); 737 } else { 738 relocate(adr.rspec(), [&] { 739 movptr(Rd, adr.target()); 740 }); 741 } 742 break; 743 } 744 case Address::base_plus_offset: { 745 Address new_adr = legitimize_address(Rd, adr); 746 if (!(new_adr.base() == Rd && new_adr.offset() == 0)) { 747 addi(Rd, new_adr.base(), new_adr.offset()); 748 } 749 break; 750 } 751 default: 752 ShouldNotReachHere(); 753 } 754 } 755 756 void MacroAssembler::la(Register Rd, Label &label) { 757 IncompressibleRegion ir(this); // the label address may be patched back. 758 wrap_label(Rd, label, &MacroAssembler::la); 759 } 760 761 void MacroAssembler::li16u(Register Rd, uint16_t imm) { 762 lui(Rd, (uint32_t)imm << 12); 763 srli(Rd, Rd, 12); 764 } 765 766 void MacroAssembler::li32(Register Rd, int32_t imm) { 767 // int32_t is in range 0x8000 0000 ~ 0x7fff ffff, and imm[31] is the sign bit 768 int64_t upper = imm, lower = imm; 769 lower = (imm << 20) >> 20; 770 upper -= lower; 771 upper = (int32_t)upper; 772 // lui Rd, imm[31:12] + imm[11] 773 lui(Rd, upper); 774 // use addiw to distinguish li32 to li64 775 addiw(Rd, Rd, lower); 776 } 777 778 void MacroAssembler::li64(Register Rd, int64_t imm) { 779 // Load upper 32 bits. upper = imm[63:32], but if imm[31] == 1 or 780 // (imm[31:20] == 0x7ff && imm[19] == 1), upper = imm[63:32] + 1. 781 int64_t lower = imm & 0xffffffff; 782 lower -= ((lower << 44) >> 44); 783 int64_t tmp_imm = ((uint64_t)(imm & 0xffffffff00000000)) + (uint64_t)lower; 784 int32_t upper = (tmp_imm - (int32_t)lower) >> 32; 785 786 // Load upper 32 bits 787 int64_t up = upper, lo = upper; 788 lo = (lo << 52) >> 52; 789 up -= lo; 790 up = (int32_t)up; 791 lui(Rd, up); 792 addi(Rd, Rd, lo); 793 794 // Load the rest 32 bits. 795 slli(Rd, Rd, 12); 796 addi(Rd, Rd, (int32_t)lower >> 20); 797 slli(Rd, Rd, 12); 798 lower = ((int32_t)imm << 12) >> 20; 799 addi(Rd, Rd, lower); 800 slli(Rd, Rd, 8); 801 lower = imm & 0xff; 802 addi(Rd, Rd, lower); 803 } 804 805 void MacroAssembler::li(Register Rd, int64_t imm) { 806 // int64_t is in range 0x8000 0000 0000 0000 ~ 0x7fff ffff ffff ffff 807 // li -> c.li 808 if (do_compress() && (is_simm6(imm) && Rd != x0)) { 809 c_li(Rd, imm); 810 return; 811 } 812 813 int shift = 12; 814 int64_t upper = imm, lower = imm; 815 // Split imm to a lower 12-bit sign-extended part and the remainder, 816 // because addi will sign-extend the lower imm. 817 lower = ((int32_t)imm << 20) >> 20; 818 upper -= lower; 819 820 // Test whether imm is a 32-bit integer. 821 if (!(((imm) & ~(int64_t)0x7fffffff) == 0 || 822 (((imm) & ~(int64_t)0x7fffffff) == ~(int64_t)0x7fffffff))) { 823 while (((upper >> shift) & 1) == 0) { shift++; } 824 upper >>= shift; 825 li(Rd, upper); 826 slli(Rd, Rd, shift); 827 if (lower != 0) { 828 addi(Rd, Rd, lower); 829 } 830 } else { 831 // 32-bit integer 832 Register hi_Rd = zr; 833 if (upper != 0) { 834 lui(Rd, (int32_t)upper); 835 hi_Rd = Rd; 836 } 837 if (lower != 0 || hi_Rd == zr) { 838 addiw(Rd, hi_Rd, lower); 839 } 840 } 841 } 842 843 #define INSN(NAME, REGISTER) \ 844 void MacroAssembler::NAME(const address dest, Register temp) { \ 845 assert_cond(dest != nullptr); \ 846 int64_t distance = dest - pc(); \ 847 if (is_simm21(distance) && ((distance % 2) == 0)) { \ 848 Assembler::jal(REGISTER, distance); \ 849 } else { \ 850 assert(temp != noreg, "expecting a register"); \ 851 int32_t offset = 0; \ 852 movptr(temp, dest, offset); \ 853 Assembler::jalr(REGISTER, temp, offset); \ 854 } \ 855 } \ 856 857 INSN(j, x0); 858 INSN(jal, x1); 859 860 #undef INSN 861 862 #define INSN(NAME, REGISTER) \ 863 void MacroAssembler::NAME(const Address &adr, Register temp) { \ 864 switch (adr.getMode()) { \ 865 case Address::literal: { \ 866 relocate(adr.rspec(), [&] { \ 867 NAME(adr.target(), temp); \ 868 }); \ 869 break; \ 870 } \ 871 case Address::base_plus_offset: { \ 872 int32_t offset = ((int32_t)adr.offset() << 20) >> 20; \ 873 la(temp, Address(adr.base(), adr.offset() - offset)); \ 874 Assembler::jalr(REGISTER, temp, offset); \ 875 break; \ 876 } \ 877 default: \ 878 ShouldNotReachHere(); \ 879 } \ 880 } 881 882 INSN(j, x0); 883 INSN(jal, x1); 884 885 #undef INSN 886 887 #define INSN(NAME) \ 888 void MacroAssembler::NAME(Register Rd, const address dest, Register temp) { \ 889 assert_cond(dest != nullptr); \ 890 int64_t distance = dest - pc(); \ 891 if (is_simm21(distance) && ((distance % 2) == 0)) { \ 892 Assembler::NAME(Rd, distance); \ 893 } else { \ 894 assert_different_registers(Rd, temp); \ 895 int32_t offset = 0; \ 896 movptr(temp, dest, offset); \ 897 jalr(Rd, temp, offset); \ 898 } \ 899 } \ 900 void MacroAssembler::NAME(Register Rd, Label &L, Register temp) { \ 901 assert_different_registers(Rd, temp); \ 902 wrap_label(Rd, L, temp, &MacroAssembler::NAME); \ 903 } 904 905 INSN(jal); 906 907 #undef INSN 908 909 #define INSN(NAME, REGISTER) \ 910 void MacroAssembler::NAME(Label &l, Register temp) { \ 911 jal(REGISTER, l, temp); \ 912 } \ 913 914 INSN(j, x0); 915 INSN(jal, x1); 916 917 #undef INSN 918 919 void MacroAssembler::wrap_label(Register Rt, Label &L, Register tmp, load_insn_by_temp insn) { 920 if (L.is_bound()) { 921 (this->*insn)(Rt, target(L), tmp); 922 } else { 923 L.add_patch_at(code(), locator()); 924 (this->*insn)(Rt, pc(), tmp); 925 } 926 } 927 928 void MacroAssembler::wrap_label(Register Rt, Label &L, jal_jalr_insn insn) { 929 if (L.is_bound()) { 930 (this->*insn)(Rt, target(L)); 931 } else { 932 L.add_patch_at(code(), locator()); 933 (this->*insn)(Rt, pc()); 934 } 935 } 936 937 void MacroAssembler::wrap_label(Register r1, Register r2, Label &L, 938 compare_and_branch_insn insn, 939 compare_and_branch_label_insn neg_insn, bool is_far) { 940 if (is_far) { 941 Label done; 942 (this->*neg_insn)(r1, r2, done, /* is_far */ false); 943 j(L); 944 bind(done); 945 } else { 946 if (L.is_bound()) { 947 (this->*insn)(r1, r2, target(L)); 948 } else { 949 L.add_patch_at(code(), locator()); 950 (this->*insn)(r1, r2, pc()); 951 } 952 } 953 } 954 955 #define INSN(NAME, NEG_INSN) \ 956 void MacroAssembler::NAME(Register Rs1, Register Rs2, Label &L, bool is_far) { \ 957 wrap_label(Rs1, Rs2, L, &MacroAssembler::NAME, &MacroAssembler::NEG_INSN, is_far); \ 958 } 959 960 INSN(beq, bne); 961 INSN(bne, beq); 962 INSN(blt, bge); 963 INSN(bge, blt); 964 INSN(bltu, bgeu); 965 INSN(bgeu, bltu); 966 967 #undef INSN 968 969 #define INSN(NAME) \ 970 void MacroAssembler::NAME##z(Register Rs, const address dest) { \ 971 NAME(Rs, zr, dest); \ 972 } \ 973 void MacroAssembler::NAME##z(Register Rs, Label &l, bool is_far) { \ 974 NAME(Rs, zr, l, is_far); \ 975 } \ 976 977 INSN(beq); 978 INSN(bne); 979 INSN(blt); 980 INSN(ble); 981 INSN(bge); 982 INSN(bgt); 983 984 #undef INSN 985 986 #define INSN(NAME, NEG_INSN) \ 987 void MacroAssembler::NAME(Register Rs, Register Rt, const address dest) { \ 988 NEG_INSN(Rt, Rs, dest); \ 989 } \ 990 void MacroAssembler::NAME(Register Rs, Register Rt, Label &l, bool is_far) { \ 991 NEG_INSN(Rt, Rs, l, is_far); \ 992 } 993 994 INSN(bgt, blt); 995 INSN(ble, bge); 996 INSN(bgtu, bltu); 997 INSN(bleu, bgeu); 998 999 #undef INSN 1000 1001 // Float compare branch instructions 1002 1003 #define INSN(NAME, FLOATCMP, BRANCH) \ 1004 void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far, bool is_unordered) { \ 1005 FLOATCMP##_s(t0, Rs1, Rs2); \ 1006 BRANCH(t0, l, is_far); \ 1007 } \ 1008 void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far, bool is_unordered) { \ 1009 FLOATCMP##_d(t0, Rs1, Rs2); \ 1010 BRANCH(t0, l, is_far); \ 1011 } 1012 1013 INSN(beq, feq, bnez); 1014 INSN(bne, feq, beqz); 1015 1016 #undef INSN 1017 1018 1019 #define INSN(NAME, FLOATCMP1, FLOATCMP2) \ 1020 void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ 1021 bool is_far, bool is_unordered) { \ 1022 if (is_unordered) { \ 1023 /* jump if either source is NaN or condition is expected */ \ 1024 FLOATCMP2##_s(t0, Rs2, Rs1); \ 1025 beqz(t0, l, is_far); \ 1026 } else { \ 1027 /* jump if no NaN in source and condition is expected */ \ 1028 FLOATCMP1##_s(t0, Rs1, Rs2); \ 1029 bnez(t0, l, is_far); \ 1030 } \ 1031 } \ 1032 void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ 1033 bool is_far, bool is_unordered) { \ 1034 if (is_unordered) { \ 1035 /* jump if either source is NaN or condition is expected */ \ 1036 FLOATCMP2##_d(t0, Rs2, Rs1); \ 1037 beqz(t0, l, is_far); \ 1038 } else { \ 1039 /* jump if no NaN in source and condition is expected */ \ 1040 FLOATCMP1##_d(t0, Rs1, Rs2); \ 1041 bnez(t0, l, is_far); \ 1042 } \ 1043 } 1044 1045 INSN(ble, fle, flt); 1046 INSN(blt, flt, fle); 1047 1048 #undef INSN 1049 1050 #define INSN(NAME, CMP) \ 1051 void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ 1052 bool is_far, bool is_unordered) { \ 1053 float_##CMP(Rs2, Rs1, l, is_far, is_unordered); \ 1054 } \ 1055 void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \ 1056 bool is_far, bool is_unordered) { \ 1057 double_##CMP(Rs2, Rs1, l, is_far, is_unordered); \ 1058 } 1059 1060 INSN(bgt, blt); 1061 INSN(bge, ble); 1062 1063 #undef INSN 1064 1065 1066 #define INSN(NAME, CSR) \ 1067 void MacroAssembler::NAME(Register Rd) { \ 1068 csrr(Rd, CSR); \ 1069 } 1070 1071 INSN(rdinstret, CSR_INSTRET); 1072 INSN(rdcycle, CSR_CYCLE); 1073 INSN(rdtime, CSR_TIME); 1074 INSN(frcsr, CSR_FCSR); 1075 INSN(frrm, CSR_FRM); 1076 INSN(frflags, CSR_FFLAGS); 1077 1078 #undef INSN 1079 1080 void MacroAssembler::csrr(Register Rd, unsigned csr) { 1081 csrrs(Rd, csr, x0); 1082 } 1083 1084 #define INSN(NAME, OPFUN) \ 1085 void MacroAssembler::NAME(unsigned csr, Register Rs) { \ 1086 OPFUN(x0, csr, Rs); \ 1087 } 1088 1089 INSN(csrw, csrrw); 1090 INSN(csrs, csrrs); 1091 INSN(csrc, csrrc); 1092 1093 #undef INSN 1094 1095 #define INSN(NAME, OPFUN) \ 1096 void MacroAssembler::NAME(unsigned csr, unsigned imm) { \ 1097 OPFUN(x0, csr, imm); \ 1098 } 1099 1100 INSN(csrwi, csrrwi); 1101 INSN(csrsi, csrrsi); 1102 INSN(csrci, csrrci); 1103 1104 #undef INSN 1105 1106 #define INSN(NAME, CSR) \ 1107 void MacroAssembler::NAME(Register Rd, Register Rs) { \ 1108 csrrw(Rd, CSR, Rs); \ 1109 } 1110 1111 INSN(fscsr, CSR_FCSR); 1112 INSN(fsrm, CSR_FRM); 1113 INSN(fsflags, CSR_FFLAGS); 1114 1115 #undef INSN 1116 1117 #define INSN(NAME) \ 1118 void MacroAssembler::NAME(Register Rs) { \ 1119 NAME(x0, Rs); \ 1120 } 1121 1122 INSN(fscsr); 1123 INSN(fsrm); 1124 INSN(fsflags); 1125 1126 #undef INSN 1127 1128 void MacroAssembler::fsrmi(Register Rd, unsigned imm) { 1129 guarantee(imm < 5, "Rounding Mode is invalid in Rounding Mode register"); 1130 csrrwi(Rd, CSR_FRM, imm); 1131 } 1132 1133 void MacroAssembler::fsflagsi(Register Rd, unsigned imm) { 1134 csrrwi(Rd, CSR_FFLAGS, imm); 1135 } 1136 1137 #define INSN(NAME) \ 1138 void MacroAssembler::NAME(unsigned imm) { \ 1139 NAME(x0, imm); \ 1140 } 1141 1142 INSN(fsrmi); 1143 INSN(fsflagsi); 1144 1145 #undef INSN 1146 1147 void MacroAssembler::push_reg(Register Rs) 1148 { 1149 addi(esp, esp, 0 - wordSize); 1150 sd(Rs, Address(esp, 0)); 1151 } 1152 1153 void MacroAssembler::pop_reg(Register Rd) 1154 { 1155 ld(Rd, Address(esp, 0)); 1156 addi(esp, esp, wordSize); 1157 } 1158 1159 int MacroAssembler::bitset_to_regs(unsigned int bitset, unsigned char* regs) { 1160 int count = 0; 1161 // Scan bitset to accumulate register pairs 1162 for (int reg = 31; reg >= 0; reg--) { 1163 if ((1U << 31) & bitset) { 1164 regs[count++] = reg; 1165 } 1166 bitset <<= 1; 1167 } 1168 return count; 1169 } 1170 1171 // Push integer registers in the bitset supplied. Don't push sp. 1172 // Return the number of words pushed 1173 int MacroAssembler::push_reg(unsigned int bitset, Register stack) { 1174 DEBUG_ONLY(int words_pushed = 0;) 1175 unsigned char regs[32]; 1176 int count = bitset_to_regs(bitset, regs); 1177 // reserve one slot to align for odd count 1178 int offset = is_even(count) ? 0 : wordSize; 1179 1180 if (count) { 1181 addi(stack, stack, -count * wordSize - offset); 1182 } 1183 for (int i = count - 1; i >= 0; i--) { 1184 sd(as_Register(regs[i]), Address(stack, (count - 1 - i) * wordSize + offset)); 1185 DEBUG_ONLY(words_pushed++;) 1186 } 1187 1188 assert(words_pushed == count, "oops, pushed != count"); 1189 1190 return count; 1191 } 1192 1193 int MacroAssembler::pop_reg(unsigned int bitset, Register stack) { 1194 DEBUG_ONLY(int words_popped = 0;) 1195 unsigned char regs[32]; 1196 int count = bitset_to_regs(bitset, regs); 1197 // reserve one slot to align for odd count 1198 int offset = is_even(count) ? 0 : wordSize; 1199 1200 for (int i = count - 1; i >= 0; i--) { 1201 ld(as_Register(regs[i]), Address(stack, (count - 1 - i) * wordSize + offset)); 1202 DEBUG_ONLY(words_popped++;) 1203 } 1204 1205 if (count) { 1206 addi(stack, stack, count * wordSize + offset); 1207 } 1208 assert(words_popped == count, "oops, popped != count"); 1209 1210 return count; 1211 } 1212 1213 // Push floating-point registers in the bitset supplied. 1214 // Return the number of words pushed 1215 int MacroAssembler::push_fp(unsigned int bitset, Register stack) { 1216 DEBUG_ONLY(int words_pushed = 0;) 1217 unsigned char regs[32]; 1218 int count = bitset_to_regs(bitset, regs); 1219 int push_slots = count + (count & 1); 1220 1221 if (count) { 1222 addi(stack, stack, -push_slots * wordSize); 1223 } 1224 1225 for (int i = count - 1; i >= 0; i--) { 1226 fsd(as_FloatRegister(regs[i]), Address(stack, (push_slots - 1 - i) * wordSize)); 1227 DEBUG_ONLY(words_pushed++;) 1228 } 1229 1230 assert(words_pushed == count, "oops, pushed(%d) != count(%d)", words_pushed, count); 1231 1232 return count; 1233 } 1234 1235 int MacroAssembler::pop_fp(unsigned int bitset, Register stack) { 1236 DEBUG_ONLY(int words_popped = 0;) 1237 unsigned char regs[32]; 1238 int count = bitset_to_regs(bitset, regs); 1239 int pop_slots = count + (count & 1); 1240 1241 for (int i = count - 1; i >= 0; i--) { 1242 fld(as_FloatRegister(regs[i]), Address(stack, (pop_slots - 1 - i) * wordSize)); 1243 DEBUG_ONLY(words_popped++;) 1244 } 1245 1246 if (count) { 1247 addi(stack, stack, pop_slots * wordSize); 1248 } 1249 1250 assert(words_popped == count, "oops, popped(%d) != count(%d)", words_popped, count); 1251 1252 return count; 1253 } 1254 1255 #ifdef COMPILER2 1256 // Push vector registers in the bitset supplied. 1257 // Return the number of words pushed 1258 int MacroAssembler::push_v(unsigned int bitset, Register stack) { 1259 int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE); 1260 1261 // Scan bitset to accumulate register pairs 1262 unsigned char regs[32]; 1263 int count = bitset_to_regs(bitset, regs); 1264 1265 for (int i = 0; i < count; i++) { 1266 sub(stack, stack, vector_size_in_bytes); 1267 vs1r_v(as_VectorRegister(regs[i]), stack); 1268 } 1269 1270 return count * vector_size_in_bytes / wordSize; 1271 } 1272 1273 int MacroAssembler::pop_v(unsigned int bitset, Register stack) { 1274 int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE); 1275 1276 // Scan bitset to accumulate register pairs 1277 unsigned char regs[32]; 1278 int count = bitset_to_regs(bitset, regs); 1279 1280 for (int i = count - 1; i >= 0; i--) { 1281 vl1r_v(as_VectorRegister(regs[i]), stack); 1282 add(stack, stack, vector_size_in_bytes); 1283 } 1284 1285 return count * vector_size_in_bytes / wordSize; 1286 } 1287 #endif // COMPILER2 1288 1289 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude) { 1290 // Push integer registers x7, x10-x17, x28-x31. 1291 push_reg(RegSet::of(x7) + RegSet::range(x10, x17) + RegSet::range(x28, x31) - exclude, sp); 1292 1293 // Push float registers f0-f7, f10-f17, f28-f31. 1294 addi(sp, sp, - wordSize * 20); 1295 int offset = 0; 1296 for (int i = 0; i < 32; i++) { 1297 if (i <= f7->encoding() || i >= f28->encoding() || (i >= f10->encoding() && i <= f17->encoding())) { 1298 fsd(as_FloatRegister(i), Address(sp, wordSize * (offset++))); 1299 } 1300 } 1301 } 1302 1303 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) { 1304 int offset = 0; 1305 for (int i = 0; i < 32; i++) { 1306 if (i <= f7->encoding() || i >= f28->encoding() || (i >= f10->encoding() && i <= f17->encoding())) { 1307 fld(as_FloatRegister(i), Address(sp, wordSize * (offset++))); 1308 } 1309 } 1310 addi(sp, sp, wordSize * 20); 1311 1312 pop_reg(RegSet::of(x7) + RegSet::range(x10, x17) + RegSet::range(x28, x31) - exclude, sp); 1313 } 1314 1315 void MacroAssembler::push_CPU_state(bool save_vectors, int vector_size_in_bytes) { 1316 // integer registers, except zr(x0) & ra(x1) & sp(x2) & gp(x3) & tp(x4) 1317 push_reg(RegSet::range(x5, x31), sp); 1318 1319 // float registers 1320 addi(sp, sp, - 32 * wordSize); 1321 for (int i = 0; i < 32; i++) { 1322 fsd(as_FloatRegister(i), Address(sp, i * wordSize)); 1323 } 1324 1325 // vector registers 1326 if (save_vectors) { 1327 sub(sp, sp, vector_size_in_bytes * VectorRegister::number_of_registers); 1328 vsetvli(t0, x0, Assembler::e64, Assembler::m8); 1329 for (int i = 0; i < VectorRegister::number_of_registers; i += 8) { 1330 add(t0, sp, vector_size_in_bytes * i); 1331 vse64_v(as_VectorRegister(i), t0); 1332 } 1333 } 1334 } 1335 1336 void MacroAssembler::pop_CPU_state(bool restore_vectors, int vector_size_in_bytes) { 1337 // vector registers 1338 if (restore_vectors) { 1339 vsetvli(t0, x0, Assembler::e64, Assembler::m8); 1340 for (int i = 0; i < VectorRegister::number_of_registers; i += 8) { 1341 vle64_v(as_VectorRegister(i), sp); 1342 add(sp, sp, vector_size_in_bytes * 8); 1343 } 1344 } 1345 1346 // float registers 1347 for (int i = 0; i < 32; i++) { 1348 fld(as_FloatRegister(i), Address(sp, i * wordSize)); 1349 } 1350 addi(sp, sp, 32 * wordSize); 1351 1352 // integer registers, except zr(x0) & ra(x1) & sp(x2) & gp(x3) & tp(x4) 1353 pop_reg(RegSet::range(x5, x31), sp); 1354 } 1355 1356 static int patch_offset_in_jal(address branch, int64_t offset) { 1357 assert(Assembler::is_simm21(offset) && ((offset % 2) == 0), 1358 "offset is too large to be patched in one jal instruction!\n"); 1359 Assembler::patch(branch, 31, 31, (offset >> 20) & 0x1); // offset[20] ==> branch[31] 1360 Assembler::patch(branch, 30, 21, (offset >> 1) & 0x3ff); // offset[10:1] ==> branch[30:21] 1361 Assembler::patch(branch, 20, 20, (offset >> 11) & 0x1); // offset[11] ==> branch[20] 1362 Assembler::patch(branch, 19, 12, (offset >> 12) & 0xff); // offset[19:12] ==> branch[19:12] 1363 return NativeInstruction::instruction_size; // only one instruction 1364 } 1365 1366 static int patch_offset_in_conditional_branch(address branch, int64_t offset) { 1367 assert(Assembler::is_simm13(offset) && ((offset % 2) == 0), 1368 "offset is too large to be patched in one beq/bge/bgeu/blt/bltu/bne instruction!\n"); 1369 Assembler::patch(branch, 31, 31, (offset >> 12) & 0x1); // offset[12] ==> branch[31] 1370 Assembler::patch(branch, 30, 25, (offset >> 5) & 0x3f); // offset[10:5] ==> branch[30:25] 1371 Assembler::patch(branch, 7, 7, (offset >> 11) & 0x1); // offset[11] ==> branch[7] 1372 Assembler::patch(branch, 11, 8, (offset >> 1) & 0xf); // offset[4:1] ==> branch[11:8] 1373 return NativeInstruction::instruction_size; // only one instruction 1374 } 1375 1376 static int patch_offset_in_pc_relative(address branch, int64_t offset) { 1377 const int PC_RELATIVE_INSTRUCTION_NUM = 2; // auipc, addi/jalr/load 1378 Assembler::patch(branch, 31, 12, ((offset + 0x800) >> 12) & 0xfffff); // Auipc. offset[31:12] ==> branch[31:12] 1379 Assembler::patch(branch + 4, 31, 20, offset & 0xfff); // Addi/Jalr/Load. offset[11:0] ==> branch[31:20] 1380 return PC_RELATIVE_INSTRUCTION_NUM * NativeInstruction::instruction_size; 1381 } 1382 1383 static int patch_addr_in_movptr(address branch, address target) { 1384 const int MOVPTR_INSTRUCTIONS_NUM = 6; // lui + addi + slli + addi + slli + addi/jalr/load 1385 int32_t lower = ((intptr_t)target << 35) >> 35; 1386 int64_t upper = ((intptr_t)target - lower) >> 29; 1387 Assembler::patch(branch + 0, 31, 12, upper & 0xfffff); // Lui. target[48:29] + target[28] ==> branch[31:12] 1388 Assembler::patch(branch + 4, 31, 20, (lower >> 17) & 0xfff); // Addi. target[28:17] ==> branch[31:20] 1389 Assembler::patch(branch + 12, 31, 20, (lower >> 6) & 0x7ff); // Addi. target[16: 6] ==> branch[31:20] 1390 Assembler::patch(branch + 20, 31, 20, lower & 0x3f); // Addi/Jalr/Load. target[ 5: 0] ==> branch[31:20] 1391 return MOVPTR_INSTRUCTIONS_NUM * NativeInstruction::instruction_size; 1392 } 1393 1394 static int patch_imm_in_li64(address branch, address target) { 1395 const int LI64_INSTRUCTIONS_NUM = 8; // lui + addi + slli + addi + slli + addi + slli + addi 1396 int64_t lower = (intptr_t)target & 0xffffffff; 1397 lower = lower - ((lower << 44) >> 44); 1398 int64_t tmp_imm = ((uint64_t)((intptr_t)target & 0xffffffff00000000)) + (uint64_t)lower; 1399 int32_t upper = (tmp_imm - (int32_t)lower) >> 32; 1400 int64_t tmp_upper = upper, tmp_lower = upper; 1401 tmp_lower = (tmp_lower << 52) >> 52; 1402 tmp_upper -= tmp_lower; 1403 tmp_upper >>= 12; 1404 // Load upper 32 bits. Upper = target[63:32], but if target[31] = 1 or (target[31:20] == 0x7ff && target[19] == 1), 1405 // upper = target[63:32] + 1. 1406 Assembler::patch(branch + 0, 31, 12, tmp_upper & 0xfffff); // Lui. 1407 Assembler::patch(branch + 4, 31, 20, tmp_lower & 0xfff); // Addi. 1408 // Load the rest 32 bits. 1409 Assembler::patch(branch + 12, 31, 20, ((int32_t)lower >> 20) & 0xfff); // Addi. 1410 Assembler::patch(branch + 20, 31, 20, (((intptr_t)target << 44) >> 52) & 0xfff); // Addi. 1411 Assembler::patch(branch + 28, 31, 20, (intptr_t)target & 0xff); // Addi. 1412 return LI64_INSTRUCTIONS_NUM * NativeInstruction::instruction_size; 1413 } 1414 1415 static int patch_imm_in_li16u(address branch, uint16_t target) { 1416 Assembler::patch(branch, 31, 12, target); // patch lui only 1417 return NativeInstruction::instruction_size; 1418 } 1419 1420 int MacroAssembler::patch_imm_in_li32(address branch, int32_t target) { 1421 const int LI32_INSTRUCTIONS_NUM = 2; // lui + addiw 1422 int64_t upper = (intptr_t)target; 1423 int32_t lower = (((int32_t)target) << 20) >> 20; 1424 upper -= lower; 1425 upper = (int32_t)upper; 1426 Assembler::patch(branch + 0, 31, 12, (upper >> 12) & 0xfffff); // Lui. 1427 Assembler::patch(branch + 4, 31, 20, lower & 0xfff); // Addiw. 1428 return LI32_INSTRUCTIONS_NUM * NativeInstruction::instruction_size; 1429 } 1430 1431 static long get_offset_of_jal(address insn_addr) { 1432 assert_cond(insn_addr != nullptr); 1433 long offset = 0; 1434 unsigned insn = Assembler::ld_instr(insn_addr); 1435 long val = (long)Assembler::sextract(insn, 31, 12); 1436 offset |= ((val >> 19) & 0x1) << 20; 1437 offset |= (val & 0xff) << 12; 1438 offset |= ((val >> 8) & 0x1) << 11; 1439 offset |= ((val >> 9) & 0x3ff) << 1; 1440 offset = (offset << 43) >> 43; 1441 return offset; 1442 } 1443 1444 static long get_offset_of_conditional_branch(address insn_addr) { 1445 long offset = 0; 1446 assert_cond(insn_addr != nullptr); 1447 unsigned insn = Assembler::ld_instr(insn_addr); 1448 offset = (long)Assembler::sextract(insn, 31, 31); 1449 offset = (offset << 12) | (((long)(Assembler::sextract(insn, 7, 7) & 0x1)) << 11); 1450 offset = offset | (((long)(Assembler::sextract(insn, 30, 25) & 0x3f)) << 5); 1451 offset = offset | (((long)(Assembler::sextract(insn, 11, 8) & 0xf)) << 1); 1452 offset = (offset << 41) >> 41; 1453 return offset; 1454 } 1455 1456 static long get_offset_of_pc_relative(address insn_addr) { 1457 long offset = 0; 1458 assert_cond(insn_addr != nullptr); 1459 offset = ((long)(Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12))) << 12; // Auipc. 1460 offset += ((long)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20)); // Addi/Jalr/Load. 1461 offset = (offset << 32) >> 32; 1462 return offset; 1463 } 1464 1465 static address get_target_of_movptr(address insn_addr) { 1466 assert_cond(insn_addr != nullptr); 1467 intptr_t target_address = (((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12)) & 0xfffff) << 29; // Lui. 1468 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20)) << 17; // Addi. 1469 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 12), 31, 20)) << 6; // Addi. 1470 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 20), 31, 20)); // Addi/Jalr/Load. 1471 return (address) target_address; 1472 } 1473 1474 static address get_target_of_li64(address insn_addr) { 1475 assert_cond(insn_addr != nullptr); 1476 intptr_t target_address = (((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12)) & 0xfffff) << 44; // Lui. 1477 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20)) << 32; // Addi. 1478 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 12), 31, 20)) << 20; // Addi. 1479 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 20), 31, 20)) << 8; // Addi. 1480 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 28), 31, 20)); // Addi. 1481 return (address)target_address; 1482 } 1483 1484 address MacroAssembler::get_target_of_li32(address insn_addr) { 1485 assert_cond(insn_addr != nullptr); 1486 intptr_t target_address = (((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12)) & 0xfffff) << 12; // Lui. 1487 target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20)); // Addiw. 1488 return (address)target_address; 1489 } 1490 1491 // Patch any kind of instruction; there may be several instructions. 1492 // Return the total length (in bytes) of the instructions. 1493 int MacroAssembler::pd_patch_instruction_size(address branch, address target) { 1494 assert_cond(branch != nullptr); 1495 int64_t offset = target - branch; 1496 if (NativeInstruction::is_jal_at(branch)) { // jal 1497 return patch_offset_in_jal(branch, offset); 1498 } else if (NativeInstruction::is_branch_at(branch)) { // beq/bge/bgeu/blt/bltu/bne 1499 return patch_offset_in_conditional_branch(branch, offset); 1500 } else if (NativeInstruction::is_pc_relative_at(branch)) { // auipc, addi/jalr/load 1501 return patch_offset_in_pc_relative(branch, offset); 1502 } else if (NativeInstruction::is_movptr_at(branch)) { // movptr 1503 return patch_addr_in_movptr(branch, target); 1504 } else if (NativeInstruction::is_li64_at(branch)) { // li64 1505 return patch_imm_in_li64(branch, target); 1506 } else if (NativeInstruction::is_li32_at(branch)) { // li32 1507 int64_t imm = (intptr_t)target; 1508 return patch_imm_in_li32(branch, (int32_t)imm); 1509 } else if (NativeInstruction::is_li16u_at(branch)) { 1510 int64_t imm = (intptr_t)target; 1511 return patch_imm_in_li16u(branch, (uint16_t)imm); 1512 } else { 1513 #ifdef ASSERT 1514 tty->print_cr("pd_patch_instruction_size: instruction 0x%x at " INTPTR_FORMAT " could not be patched!\n", 1515 Assembler::ld_instr(branch), p2i(branch)); 1516 Disassembler::decode(branch - 16, branch + 16); 1517 #endif 1518 ShouldNotReachHere(); 1519 return -1; 1520 } 1521 } 1522 1523 address MacroAssembler::target_addr_for_insn(address insn_addr) { 1524 long offset = 0; 1525 assert_cond(insn_addr != nullptr); 1526 if (NativeInstruction::is_jal_at(insn_addr)) { // jal 1527 offset = get_offset_of_jal(insn_addr); 1528 } else if (NativeInstruction::is_branch_at(insn_addr)) { // beq/bge/bgeu/blt/bltu/bne 1529 offset = get_offset_of_conditional_branch(insn_addr); 1530 } else if (NativeInstruction::is_pc_relative_at(insn_addr)) { // auipc, addi/jalr/load 1531 offset = get_offset_of_pc_relative(insn_addr); 1532 } else if (NativeInstruction::is_movptr_at(insn_addr)) { // movptr 1533 return get_target_of_movptr(insn_addr); 1534 } else if (NativeInstruction::is_li64_at(insn_addr)) { // li64 1535 return get_target_of_li64(insn_addr); 1536 } else if (NativeInstruction::is_li32_at(insn_addr)) { // li32 1537 return get_target_of_li32(insn_addr); 1538 } else { 1539 ShouldNotReachHere(); 1540 } 1541 return address(((uintptr_t)insn_addr + offset)); 1542 } 1543 1544 int MacroAssembler::patch_oop(address insn_addr, address o) { 1545 // OOPs are either narrow (32 bits) or wide (48 bits). We encode 1546 // narrow OOPs by setting the upper 16 bits in the first 1547 // instruction. 1548 if (NativeInstruction::is_li32_at(insn_addr)) { 1549 // Move narrow OOP 1550 uint32_t n = CompressedOops::narrow_oop_value(cast_to_oop(o)); 1551 return patch_imm_in_li32(insn_addr, (int32_t)n); 1552 } else if (NativeInstruction::is_movptr_at(insn_addr)) { 1553 // Move wide OOP 1554 return patch_addr_in_movptr(insn_addr, o); 1555 } 1556 ShouldNotReachHere(); 1557 return -1; 1558 } 1559 1560 void MacroAssembler::reinit_heapbase() { 1561 if (UseCompressedOops) { 1562 if (Universe::is_fully_initialized()) { 1563 mv(xheapbase, CompressedOops::ptrs_base()); 1564 } else { 1565 ExternalAddress target(CompressedOops::ptrs_base_addr()); 1566 relocate(target.rspec(), [&] { 1567 int32_t offset; 1568 la_patchable(xheapbase, target, offset); 1569 ld(xheapbase, Address(xheapbase, offset)); 1570 }); 1571 } 1572 } 1573 } 1574 1575 void MacroAssembler::movptr(Register Rd, address addr, int32_t &offset) { 1576 int64_t imm64 = (int64_t)addr; 1577 #ifndef PRODUCT 1578 { 1579 char buffer[64]; 1580 snprintf(buffer, sizeof(buffer), "0x%" PRIx64, imm64); 1581 block_comment(buffer); 1582 } 1583 #endif 1584 assert((uintptr_t)imm64 < (1ull << 48), "48-bit overflow in address constant"); 1585 // Load upper 31 bits 1586 int64_t imm = imm64 >> 17; 1587 int64_t upper = imm, lower = imm; 1588 lower = (lower << 52) >> 52; 1589 upper -= lower; 1590 upper = (int32_t)upper; 1591 lui(Rd, upper); 1592 addi(Rd, Rd, lower); 1593 1594 // Load the rest 17 bits. 1595 slli(Rd, Rd, 11); 1596 addi(Rd, Rd, (imm64 >> 6) & 0x7ff); 1597 slli(Rd, Rd, 6); 1598 1599 // This offset will be used by following jalr/ld. 1600 offset = imm64 & 0x3f; 1601 } 1602 1603 void MacroAssembler::add(Register Rd, Register Rn, int64_t increment, Register temp) { 1604 if (is_simm12(increment)) { 1605 addi(Rd, Rn, increment); 1606 } else { 1607 assert_different_registers(Rn, temp); 1608 li(temp, increment); 1609 add(Rd, Rn, temp); 1610 } 1611 } 1612 1613 void MacroAssembler::addw(Register Rd, Register Rn, int32_t increment, Register temp) { 1614 if (is_simm12(increment)) { 1615 addiw(Rd, Rn, increment); 1616 } else { 1617 assert_different_registers(Rn, temp); 1618 li(temp, increment); 1619 addw(Rd, Rn, temp); 1620 } 1621 } 1622 1623 void MacroAssembler::sub(Register Rd, Register Rn, int64_t decrement, Register temp) { 1624 if (is_simm12(-decrement)) { 1625 addi(Rd, Rn, -decrement); 1626 } else { 1627 assert_different_registers(Rn, temp); 1628 li(temp, decrement); 1629 sub(Rd, Rn, temp); 1630 } 1631 } 1632 1633 void MacroAssembler::subw(Register Rd, Register Rn, int32_t decrement, Register temp) { 1634 if (is_simm12(-decrement)) { 1635 addiw(Rd, Rn, -decrement); 1636 } else { 1637 assert_different_registers(Rn, temp); 1638 li(temp, decrement); 1639 subw(Rd, Rn, temp); 1640 } 1641 } 1642 1643 void MacroAssembler::andrw(Register Rd, Register Rs1, Register Rs2) { 1644 andr(Rd, Rs1, Rs2); 1645 sign_extend(Rd, Rd, 32); 1646 } 1647 1648 void MacroAssembler::orrw(Register Rd, Register Rs1, Register Rs2) { 1649 orr(Rd, Rs1, Rs2); 1650 sign_extend(Rd, Rd, 32); 1651 } 1652 1653 void MacroAssembler::xorrw(Register Rd, Register Rs1, Register Rs2) { 1654 xorr(Rd, Rs1, Rs2); 1655 sign_extend(Rd, Rd, 32); 1656 } 1657 1658 // Rd = Rs1 & (~Rd2) 1659 void MacroAssembler::andn(Register Rd, Register Rs1, Register Rs2) { 1660 if (UseZbb) { 1661 Assembler::andn(Rd, Rs1, Rs2); 1662 return; 1663 } 1664 1665 notr(Rd, Rs2); 1666 andr(Rd, Rs1, Rd); 1667 } 1668 1669 // Rd = Rs1 | (~Rd2) 1670 void MacroAssembler::orn(Register Rd, Register Rs1, Register Rs2) { 1671 if (UseZbb) { 1672 Assembler::orn(Rd, Rs1, Rs2); 1673 return; 1674 } 1675 1676 notr(Rd, Rs2); 1677 orr(Rd, Rs1, Rd); 1678 } 1679 1680 // Note: load_unsigned_short used to be called load_unsigned_word. 1681 int MacroAssembler::load_unsigned_short(Register dst, Address src) { 1682 int off = offset(); 1683 lhu(dst, src); 1684 return off; 1685 } 1686 1687 int MacroAssembler::load_unsigned_byte(Register dst, Address src) { 1688 int off = offset(); 1689 lbu(dst, src); 1690 return off; 1691 } 1692 1693 int MacroAssembler::load_signed_short(Register dst, Address src) { 1694 int off = offset(); 1695 lh(dst, src); 1696 return off; 1697 } 1698 1699 int MacroAssembler::load_signed_byte(Register dst, Address src) { 1700 int off = offset(); 1701 lb(dst, src); 1702 return off; 1703 } 1704 1705 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed) { 1706 switch (size_in_bytes) { 1707 case 8: ld(dst, src); break; 1708 case 4: is_signed ? lw(dst, src) : lwu(dst, src); break; 1709 case 2: is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break; 1710 case 1: is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break; 1711 default: ShouldNotReachHere(); 1712 } 1713 } 1714 1715 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes) { 1716 switch (size_in_bytes) { 1717 case 8: sd(src, dst); break; 1718 case 4: sw(src, dst); break; 1719 case 2: sh(src, dst); break; 1720 case 1: sb(src, dst); break; 1721 default: ShouldNotReachHere(); 1722 } 1723 } 1724 1725 // granularity is 1 OR 2 bytes per load. dst and src.base() allowed to be the same register 1726 void MacroAssembler::load_short_misaligned(Register dst, Address src, Register tmp, bool is_signed, int granularity) { 1727 if (granularity != 1 && granularity != 2) { 1728 ShouldNotReachHere(); 1729 } 1730 if (AvoidUnalignedAccesses && (granularity != 2)) { 1731 assert_different_registers(dst, tmp); 1732 assert_different_registers(tmp, src.base()); 1733 is_signed ? lb(tmp, Address(src.base(), src.offset() + 1)) : lbu(tmp, Address(src.base(), src.offset() + 1)); 1734 slli(tmp, tmp, 8); 1735 lbu(dst, src); 1736 add(dst, dst, tmp); 1737 } else { 1738 is_signed ? lh(dst, src) : lhu(dst, src); 1739 } 1740 } 1741 1742 // granularity is 1, 2 OR 4 bytes per load, if granularity 2 or 4 then dst and src.base() allowed to be the same register 1743 void MacroAssembler::load_int_misaligned(Register dst, Address src, Register tmp, bool is_signed, int granularity) { 1744 if (AvoidUnalignedAccesses && (granularity != 4)) { 1745 switch(granularity) { 1746 case 1: 1747 assert_different_registers(dst, tmp, src.base()); 1748 lbu(dst, src); 1749 lbu(tmp, Address(src.base(), src.offset() + 1)); 1750 slli(tmp, tmp, 8); 1751 add(dst, dst, tmp); 1752 lbu(tmp, Address(src.base(), src.offset() + 2)); 1753 slli(tmp, tmp, 16); 1754 add(dst, dst, tmp); 1755 is_signed ? lb(tmp, Address(src.base(), src.offset() + 3)) : lbu(tmp, Address(src.base(), src.offset() + 3)); 1756 slli(tmp, tmp, 24); 1757 add(dst, dst, tmp); 1758 break; 1759 case 2: 1760 assert_different_registers(dst, tmp); 1761 assert_different_registers(tmp, src.base()); 1762 is_signed ? lh(tmp, Address(src.base(), src.offset() + 2)) : lhu(tmp, Address(src.base(), src.offset() + 2)); 1763 slli(tmp, tmp, 16); 1764 lhu(dst, src); 1765 add(dst, dst, tmp); 1766 break; 1767 default: 1768 ShouldNotReachHere(); 1769 } 1770 } else { 1771 is_signed ? lw(dst, src) : lwu(dst, src); 1772 } 1773 } 1774 1775 // granularity is 1, 2, 4 or 8 bytes per load, if granularity 4 or 8 then dst and src.base() allowed to be same register 1776 void MacroAssembler::load_long_misaligned(Register dst, Address src, Register tmp, int granularity) { 1777 if (AvoidUnalignedAccesses && (granularity != 8)) { 1778 switch(granularity){ 1779 case 1: 1780 assert_different_registers(dst, tmp, src.base()); 1781 lbu(dst, src); 1782 lbu(tmp, Address(src.base(), src.offset() + 1)); 1783 slli(tmp, tmp, 8); 1784 add(dst, dst, tmp); 1785 lbu(tmp, Address(src.base(), src.offset() + 2)); 1786 slli(tmp, tmp, 16); 1787 add(dst, dst, tmp); 1788 lbu(tmp, Address(src.base(), src.offset() + 3)); 1789 slli(tmp, tmp, 24); 1790 add(dst, dst, tmp); 1791 lbu(tmp, Address(src.base(), src.offset() + 4)); 1792 slli(tmp, tmp, 32); 1793 add(dst, dst, tmp); 1794 lbu(tmp, Address(src.base(), src.offset() + 5)); 1795 slli(tmp, tmp, 40); 1796 add(dst, dst, tmp); 1797 lbu(tmp, Address(src.base(), src.offset() + 6)); 1798 slli(tmp, tmp, 48); 1799 add(dst, dst, tmp); 1800 lbu(tmp, Address(src.base(), src.offset() + 7)); 1801 slli(tmp, tmp, 56); 1802 add(dst, dst, tmp); 1803 break; 1804 case 2: 1805 assert_different_registers(dst, tmp, src.base()); 1806 lhu(dst, src); 1807 lhu(tmp, Address(src.base(), src.offset() + 2)); 1808 slli(tmp, tmp, 16); 1809 add(dst, dst, tmp); 1810 lhu(tmp, Address(src.base(), src.offset() + 4)); 1811 slli(tmp, tmp, 32); 1812 add(dst, dst, tmp); 1813 lhu(tmp, Address(src.base(), src.offset() + 6)); 1814 slli(tmp, tmp, 48); 1815 add(dst, dst, tmp); 1816 break; 1817 case 4: 1818 assert_different_registers(dst, tmp); 1819 assert_different_registers(tmp, src.base()); 1820 lwu(tmp, Address(src.base(), src.offset() + 4)); 1821 slli(tmp, tmp, 32); 1822 lwu(dst, src); 1823 add(dst, dst, tmp); 1824 break; 1825 default: 1826 ShouldNotReachHere(); 1827 } 1828 } else { 1829 ld(dst, src); 1830 } 1831 } 1832 1833 1834 // reverse bytes in halfword in lower 16 bits and sign-extend 1835 // Rd[15:0] = Rs[7:0] Rs[15:8] (sign-extend to 64 bits) 1836 void MacroAssembler::revb_h_h(Register Rd, Register Rs, Register tmp) { 1837 if (UseZbb) { 1838 rev8(Rd, Rs); 1839 srai(Rd, Rd, 48); 1840 return; 1841 } 1842 assert_different_registers(Rs, tmp); 1843 assert_different_registers(Rd, tmp); 1844 srli(tmp, Rs, 8); 1845 andi(tmp, tmp, 0xFF); 1846 slli(Rd, Rs, 56); 1847 srai(Rd, Rd, 48); // sign-extend 1848 orr(Rd, Rd, tmp); 1849 } 1850 1851 // reverse bytes in lower word and sign-extend 1852 // Rd[31:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] (sign-extend to 64 bits) 1853 void MacroAssembler::revb_w_w(Register Rd, Register Rs, Register tmp1, Register tmp2) { 1854 if (UseZbb) { 1855 rev8(Rd, Rs); 1856 srai(Rd, Rd, 32); 1857 return; 1858 } 1859 assert_different_registers(Rs, tmp1, tmp2); 1860 assert_different_registers(Rd, tmp1, tmp2); 1861 revb_h_w_u(Rd, Rs, tmp1, tmp2); 1862 slli(tmp2, Rd, 48); 1863 srai(tmp2, tmp2, 32); // sign-extend 1864 srli(Rd, Rd, 16); 1865 orr(Rd, Rd, tmp2); 1866 } 1867 1868 // reverse bytes in halfword in lower 16 bits and zero-extend 1869 // Rd[15:0] = Rs[7:0] Rs[15:8] (zero-extend to 64 bits) 1870 void MacroAssembler::revb_h_h_u(Register Rd, Register Rs, Register tmp) { 1871 if (UseZbb) { 1872 rev8(Rd, Rs); 1873 srli(Rd, Rd, 48); 1874 return; 1875 } 1876 assert_different_registers(Rs, tmp); 1877 assert_different_registers(Rd, tmp); 1878 srli(tmp, Rs, 8); 1879 andi(tmp, tmp, 0xFF); 1880 andi(Rd, Rs, 0xFF); 1881 slli(Rd, Rd, 8); 1882 orr(Rd, Rd, tmp); 1883 } 1884 1885 // reverse bytes in halfwords in lower 32 bits and zero-extend 1886 // Rd[31:0] = Rs[23:16] Rs[31:24] Rs[7:0] Rs[15:8] (zero-extend to 64 bits) 1887 void MacroAssembler::revb_h_w_u(Register Rd, Register Rs, Register tmp1, Register tmp2) { 1888 if (UseZbb) { 1889 rev8(Rd, Rs); 1890 rori(Rd, Rd, 32); 1891 roriw(Rd, Rd, 16); 1892 zero_extend(Rd, Rd, 32); 1893 return; 1894 } 1895 assert_different_registers(Rs, tmp1, tmp2); 1896 assert_different_registers(Rd, tmp1, tmp2); 1897 srli(tmp2, Rs, 16); 1898 revb_h_h_u(tmp2, tmp2, tmp1); 1899 revb_h_h_u(Rd, Rs, tmp1); 1900 slli(tmp2, tmp2, 16); 1901 orr(Rd, Rd, tmp2); 1902 } 1903 1904 // This method is only used for revb_h 1905 // Rd = Rs[47:0] Rs[55:48] Rs[63:56] 1906 void MacroAssembler::revb_h_helper(Register Rd, Register Rs, Register tmp1, Register tmp2) { 1907 assert_different_registers(Rs, tmp1, tmp2); 1908 assert_different_registers(Rd, tmp1); 1909 srli(tmp1, Rs, 48); 1910 andi(tmp2, tmp1, 0xFF); 1911 slli(tmp2, tmp2, 8); 1912 srli(tmp1, tmp1, 8); 1913 orr(tmp1, tmp1, tmp2); 1914 slli(Rd, Rs, 16); 1915 orr(Rd, Rd, tmp1); 1916 } 1917 1918 // reverse bytes in each halfword 1919 // Rd[63:0] = Rs[55:48] Rs[63:56] Rs[39:32] Rs[47:40] Rs[23:16] Rs[31:24] Rs[7:0] Rs[15:8] 1920 void MacroAssembler::revb_h(Register Rd, Register Rs, Register tmp1, Register tmp2) { 1921 if (UseZbb) { 1922 assert_different_registers(Rs, tmp1); 1923 assert_different_registers(Rd, tmp1); 1924 rev8(Rd, Rs); 1925 zero_extend(tmp1, Rd, 32); 1926 roriw(tmp1, tmp1, 16); 1927 slli(tmp1, tmp1, 32); 1928 srli(Rd, Rd, 32); 1929 roriw(Rd, Rd, 16); 1930 zero_extend(Rd, Rd, 32); 1931 orr(Rd, Rd, tmp1); 1932 return; 1933 } 1934 assert_different_registers(Rs, tmp1, tmp2); 1935 assert_different_registers(Rd, tmp1, tmp2); 1936 revb_h_helper(Rd, Rs, tmp1, tmp2); 1937 for (int i = 0; i < 3; ++i) { 1938 revb_h_helper(Rd, Rd, tmp1, tmp2); 1939 } 1940 } 1941 1942 // reverse bytes in each word 1943 // Rd[63:0] = Rs[39:32] Rs[47:40] Rs[55:48] Rs[63:56] Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] 1944 void MacroAssembler::revb_w(Register Rd, Register Rs, Register tmp1, Register tmp2) { 1945 if (UseZbb) { 1946 rev8(Rd, Rs); 1947 rori(Rd, Rd, 32); 1948 return; 1949 } 1950 assert_different_registers(Rs, tmp1, tmp2); 1951 assert_different_registers(Rd, tmp1, tmp2); 1952 revb(Rd, Rs, tmp1, tmp2); 1953 ror_imm(Rd, Rd, 32); 1954 } 1955 1956 // reverse bytes in doubleword 1957 // Rd[63:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] Rs[39:32] Rs[47,40] Rs[55,48] Rs[63:56] 1958 void MacroAssembler::revb(Register Rd, Register Rs, Register tmp1, Register tmp2) { 1959 if (UseZbb) { 1960 rev8(Rd, Rs); 1961 return; 1962 } 1963 assert_different_registers(Rs, tmp1, tmp2); 1964 assert_different_registers(Rd, tmp1, tmp2); 1965 andi(tmp1, Rs, 0xFF); 1966 slli(tmp1, tmp1, 8); 1967 for (int step = 8; step < 56; step += 8) { 1968 srli(tmp2, Rs, step); 1969 andi(tmp2, tmp2, 0xFF); 1970 orr(tmp1, tmp1, tmp2); 1971 slli(tmp1, tmp1, 8); 1972 } 1973 srli(Rd, Rs, 56); 1974 andi(Rd, Rd, 0xFF); 1975 orr(Rd, tmp1, Rd); 1976 } 1977 1978 // rotate right with shift bits 1979 void MacroAssembler::ror_imm(Register dst, Register src, uint32_t shift, Register tmp) 1980 { 1981 if (UseZbb) { 1982 rori(dst, src, shift); 1983 return; 1984 } 1985 1986 assert_different_registers(dst, tmp); 1987 assert_different_registers(src, tmp); 1988 assert(shift < 64, "shift amount must be < 64"); 1989 slli(tmp, src, 64 - shift); 1990 srli(dst, src, shift); 1991 orr(dst, dst, tmp); 1992 } 1993 1994 // rotate left with shift bits, 32-bit version 1995 void MacroAssembler::rolw_imm(Register dst, Register src, uint32_t shift, Register tmp) { 1996 if (UseZbb) { 1997 // no roliw available 1998 roriw(dst, src, 32 - shift); 1999 return; 2000 } 2001 2002 assert_different_registers(dst, tmp); 2003 assert_different_registers(src, tmp); 2004 assert(shift < 32, "shift amount must be < 32"); 2005 srliw(tmp, src, 32 - shift); 2006 slliw(dst, src, shift); 2007 orr(dst, dst, tmp); 2008 } 2009 2010 void MacroAssembler::andi(Register Rd, Register Rn, int64_t imm, Register tmp) { 2011 if (is_simm12(imm)) { 2012 and_imm12(Rd, Rn, imm); 2013 } else { 2014 assert_different_registers(Rn, tmp); 2015 mv(tmp, imm); 2016 andr(Rd, Rn, tmp); 2017 } 2018 } 2019 2020 void MacroAssembler::orptr(Address adr, RegisterOrConstant src, Register tmp1, Register tmp2) { 2021 ld(tmp1, adr); 2022 if (src.is_register()) { 2023 orr(tmp1, tmp1, src.as_register()); 2024 } else { 2025 if (is_simm12(src.as_constant())) { 2026 ori(tmp1, tmp1, src.as_constant()); 2027 } else { 2028 assert_different_registers(tmp1, tmp2); 2029 mv(tmp2, src.as_constant()); 2030 orr(tmp1, tmp1, tmp2); 2031 } 2032 } 2033 sd(tmp1, adr); 2034 } 2035 2036 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp1, Register tmp2, Label &L) { 2037 assert_different_registers(oop, trial_klass, tmp1, tmp2); 2038 if (UseCompressedClassPointers) { 2039 lwu(tmp1, Address(oop, oopDesc::klass_offset_in_bytes())); 2040 if (CompressedKlassPointers::base() == nullptr) { 2041 slli(tmp1, tmp1, CompressedKlassPointers::shift()); 2042 beq(trial_klass, tmp1, L); 2043 return; 2044 } 2045 decode_klass_not_null(tmp1, tmp2); 2046 } else { 2047 ld(tmp1, Address(oop, oopDesc::klass_offset_in_bytes())); 2048 } 2049 beq(trial_klass, tmp1, L); 2050 } 2051 2052 // Move an oop into a register. 2053 void MacroAssembler::movoop(Register dst, jobject obj) { 2054 int oop_index; 2055 if (obj == nullptr) { 2056 oop_index = oop_recorder()->allocate_oop_index(obj); 2057 } else { 2058 #ifdef ASSERT 2059 { 2060 ThreadInVMfromUnknown tiv; 2061 assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop"); 2062 } 2063 #endif 2064 oop_index = oop_recorder()->find_index(obj); 2065 } 2066 RelocationHolder rspec = oop_Relocation::spec(oop_index); 2067 2068 if (BarrierSet::barrier_set()->barrier_set_assembler()->supports_instruction_patching()) { 2069 mv(dst, Address((address)obj, rspec)); 2070 } else { 2071 address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address 2072 ld_constant(dst, Address(dummy, rspec)); 2073 } 2074 } 2075 2076 // Move a metadata address into a register. 2077 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) { 2078 int oop_index; 2079 if (obj == nullptr) { 2080 oop_index = oop_recorder()->allocate_metadata_index(obj); 2081 } else { 2082 oop_index = oop_recorder()->find_index(obj); 2083 } 2084 RelocationHolder rspec = metadata_Relocation::spec(oop_index); 2085 mv(dst, Address((address)obj, rspec)); 2086 } 2087 2088 // Writes to stack successive pages until offset reached to check for 2089 // stack overflow + shadow pages. This clobbers tmp. 2090 void MacroAssembler::bang_stack_size(Register size, Register tmp) { 2091 assert_different_registers(tmp, size, t0); 2092 // Bang stack for total size given plus shadow page size. 2093 // Bang one page at a time because large size can bang beyond yellow and 2094 // red zones. 2095 mv(t0, (int)os::vm_page_size()); 2096 Label loop; 2097 bind(loop); 2098 sub(tmp, sp, t0); 2099 subw(size, size, t0); 2100 sd(size, Address(tmp)); 2101 bgtz(size, loop); 2102 2103 // Bang down shadow pages too. 2104 // At this point, (tmp-0) is the last address touched, so don't 2105 // touch it again. (It was touched as (tmp-pagesize) but then tmp 2106 // was post-decremented.) Skip this address by starting at i=1, and 2107 // touch a few more pages below. N.B. It is important to touch all 2108 // the way down to and including i=StackShadowPages. 2109 for (int i = 0; i < (int)(StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()) - 1; i++) { 2110 // this could be any sized move but this is can be a debugging crumb 2111 // so the bigger the better. 2112 sub(tmp, tmp, (int)os::vm_page_size()); 2113 sd(size, Address(tmp, 0)); 2114 } 2115 } 2116 2117 SkipIfEqual::SkipIfEqual(MacroAssembler* masm, const bool* flag_addr, bool value) { 2118 int32_t offset = 0; 2119 _masm = masm; 2120 ExternalAddress target((address)flag_addr); 2121 _masm->relocate(target.rspec(), [&] { 2122 int32_t offset; 2123 _masm->la_patchable(t0, target, offset); 2124 _masm->lbu(t0, Address(t0, offset)); 2125 }); 2126 if (value) { 2127 _masm->bnez(t0, _label); 2128 } else { 2129 _masm->beqz(t0, _label); 2130 } 2131 } 2132 2133 SkipIfEqual::~SkipIfEqual() { 2134 _masm->bind(_label); 2135 _masm = nullptr; 2136 } 2137 2138 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, Register tmp2) { 2139 const int mirror_offset = in_bytes(Klass::java_mirror_offset()); 2140 ld(dst, Address(xmethod, Method::const_offset())); 2141 ld(dst, Address(dst, ConstMethod::constants_offset())); 2142 ld(dst, Address(dst, ConstantPool::pool_holder_offset())); 2143 ld(dst, Address(dst, mirror_offset)); 2144 resolve_oop_handle(dst, tmp1, tmp2); 2145 } 2146 2147 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) { 2148 // OopHandle::resolve is an indirection. 2149 assert_different_registers(result, tmp1, tmp2); 2150 access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp1, tmp2); 2151 } 2152 2153 // ((WeakHandle)result).resolve() 2154 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2) { 2155 assert_different_registers(result, tmp1, tmp2); 2156 Label resolved; 2157 2158 // A null weak handle resolves to null. 2159 beqz(result, resolved); 2160 2161 // Only 64 bit platforms support GCs that require a tmp register 2162 // Only IN_HEAP loads require a thread_tmp register 2163 // WeakHandle::resolve is an indirection like jweak. 2164 access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, 2165 result, Address(result), tmp1, tmp2); 2166 bind(resolved); 2167 } 2168 2169 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators, 2170 Register dst, Address src, 2171 Register tmp1, Register tmp2) { 2172 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 2173 decorators = AccessInternal::decorator_fixup(decorators, type); 2174 bool as_raw = (decorators & AS_RAW) != 0; 2175 if (as_raw) { 2176 bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2); 2177 } else { 2178 bs->load_at(this, decorators, type, dst, src, tmp1, tmp2); 2179 } 2180 } 2181 2182 void MacroAssembler::null_check(Register reg, int offset) { 2183 if (needs_explicit_null_check(offset)) { 2184 // provoke OS null exception if reg is null by 2185 // accessing M[reg] w/o changing any registers 2186 // NOTE: this is plenty to provoke a segv 2187 ld(zr, Address(reg, 0)); 2188 } else { 2189 // nothing to do, (later) access of M[reg + offset] 2190 // will provoke OS null exception if reg is null 2191 } 2192 } 2193 2194 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators, 2195 Address dst, Register val, 2196 Register tmp1, Register tmp2, Register tmp3) { 2197 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 2198 decorators = AccessInternal::decorator_fixup(decorators, type); 2199 bool as_raw = (decorators & AS_RAW) != 0; 2200 if (as_raw) { 2201 bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 2202 } else { 2203 bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3); 2204 } 2205 } 2206 2207 // Algorithm must match CompressedOops::encode. 2208 void MacroAssembler::encode_heap_oop(Register d, Register s) { 2209 verify_oop_msg(s, "broken oop in encode_heap_oop"); 2210 if (CompressedOops::base() == nullptr) { 2211 if (CompressedOops::shift() != 0) { 2212 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 2213 srli(d, s, LogMinObjAlignmentInBytes); 2214 } else { 2215 mv(d, s); 2216 } 2217 } else { 2218 Label notNull; 2219 sub(d, s, xheapbase); 2220 bgez(d, notNull); 2221 mv(d, zr); 2222 bind(notNull); 2223 if (CompressedOops::shift() != 0) { 2224 assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 2225 srli(d, d, CompressedOops::shift()); 2226 } 2227 } 2228 } 2229 2230 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) { 2231 assert_different_registers(dst, tmp); 2232 assert_different_registers(src, tmp); 2233 if (UseCompressedClassPointers) { 2234 lwu(dst, Address(src, oopDesc::klass_offset_in_bytes())); 2235 decode_klass_not_null(dst, tmp); 2236 } else { 2237 ld(dst, Address(src, oopDesc::klass_offset_in_bytes())); 2238 } 2239 } 2240 2241 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) { 2242 // FIXME: Should this be a store release? concurrent gcs assumes 2243 // klass length is valid if klass field is not null. 2244 if (UseCompressedClassPointers) { 2245 encode_klass_not_null(src, tmp); 2246 sw(src, Address(dst, oopDesc::klass_offset_in_bytes())); 2247 } else { 2248 sd(src, Address(dst, oopDesc::klass_offset_in_bytes())); 2249 } 2250 } 2251 2252 void MacroAssembler::store_klass_gap(Register dst, Register src) { 2253 if (UseCompressedClassPointers) { 2254 // Store to klass gap in destination 2255 sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes())); 2256 } 2257 } 2258 2259 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) { 2260 assert_different_registers(r, tmp); 2261 decode_klass_not_null(r, r, tmp); 2262 } 2263 2264 void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register tmp) { 2265 assert(UseCompressedClassPointers, "should only be used for compressed headers"); 2266 2267 if (CompressedKlassPointers::base() == nullptr) { 2268 if (CompressedKlassPointers::shift() != 0) { 2269 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 2270 slli(dst, src, LogKlassAlignmentInBytes); 2271 } else { 2272 mv(dst, src); 2273 } 2274 return; 2275 } 2276 2277 Register xbase = dst; 2278 if (dst == src) { 2279 xbase = tmp; 2280 } 2281 2282 assert_different_registers(src, xbase); 2283 mv(xbase, (uintptr_t)CompressedKlassPointers::base()); 2284 2285 if (CompressedKlassPointers::shift() != 0) { 2286 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 2287 assert_different_registers(t0, xbase); 2288 shadd(dst, src, xbase, t0, LogKlassAlignmentInBytes); 2289 } else { 2290 add(dst, xbase, src); 2291 } 2292 } 2293 2294 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) { 2295 assert_different_registers(r, tmp); 2296 encode_klass_not_null(r, r, tmp); 2297 } 2298 2299 void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register tmp) { 2300 assert(UseCompressedClassPointers, "should only be used for compressed headers"); 2301 2302 if (CompressedKlassPointers::base() == nullptr) { 2303 if (CompressedKlassPointers::shift() != 0) { 2304 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 2305 srli(dst, src, LogKlassAlignmentInBytes); 2306 } else { 2307 mv(dst, src); 2308 } 2309 return; 2310 } 2311 2312 if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0 && 2313 CompressedKlassPointers::shift() == 0) { 2314 zero_extend(dst, src, 32); 2315 return; 2316 } 2317 2318 Register xbase = dst; 2319 if (dst == src) { 2320 xbase = tmp; 2321 } 2322 2323 assert_different_registers(src, xbase); 2324 mv(xbase, (uintptr_t)CompressedKlassPointers::base()); 2325 sub(dst, src, xbase); 2326 if (CompressedKlassPointers::shift() != 0) { 2327 assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong"); 2328 srli(dst, dst, LogKlassAlignmentInBytes); 2329 } 2330 } 2331 2332 void MacroAssembler::decode_heap_oop_not_null(Register r) { 2333 decode_heap_oop_not_null(r, r); 2334 } 2335 2336 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) { 2337 assert(UseCompressedOops, "should only be used for compressed headers"); 2338 assert(Universe::heap() != nullptr, "java heap should be initialized"); 2339 // Cannot assert, unverified entry point counts instructions (see .ad file) 2340 // vtableStubs also counts instructions in pd_code_size_limit. 2341 // Also do not verify_oop as this is called by verify_oop. 2342 if (CompressedOops::shift() != 0) { 2343 assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong"); 2344 slli(dst, src, LogMinObjAlignmentInBytes); 2345 if (CompressedOops::base() != nullptr) { 2346 add(dst, xheapbase, dst); 2347 } 2348 } else { 2349 assert(CompressedOops::base() == nullptr, "sanity"); 2350 mv(dst, src); 2351 } 2352 } 2353 2354 void MacroAssembler::decode_heap_oop(Register d, Register s) { 2355 if (CompressedOops::base() == nullptr) { 2356 if (CompressedOops::shift() != 0 || d != s) { 2357 slli(d, s, CompressedOops::shift()); 2358 } 2359 } else { 2360 Label done; 2361 mv(d, s); 2362 beqz(s, done); 2363 shadd(d, s, xheapbase, d, LogMinObjAlignmentInBytes); 2364 bind(done); 2365 } 2366 verify_oop_msg(d, "broken oop in decode_heap_oop"); 2367 } 2368 2369 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1, 2370 Register tmp2, Register tmp3, DecoratorSet decorators) { 2371 access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3); 2372 } 2373 2374 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1, 2375 Register tmp2, DecoratorSet decorators) { 2376 access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2); 2377 } 2378 2379 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1, 2380 Register tmp2, DecoratorSet decorators) { 2381 access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL, dst, src, tmp1, tmp2); 2382 } 2383 2384 // Used for storing nulls. 2385 void MacroAssembler::store_heap_oop_null(Address dst) { 2386 access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg); 2387 } 2388 2389 int MacroAssembler::corrected_idivl(Register result, Register rs1, Register rs2, 2390 bool want_remainder) 2391 { 2392 // Full implementation of Java idiv and irem. The function 2393 // returns the (pc) offset of the div instruction - may be needed 2394 // for implicit exceptions. 2395 // 2396 // input : rs1: dividend 2397 // rs2: divisor 2398 // 2399 // result: either 2400 // quotient (= rs1 idiv rs2) 2401 // remainder (= rs1 irem rs2) 2402 2403 2404 int idivl_offset = offset(); 2405 if (!want_remainder) { 2406 divw(result, rs1, rs2); 2407 } else { 2408 remw(result, rs1, rs2); // result = rs1 % rs2; 2409 } 2410 return idivl_offset; 2411 } 2412 2413 int MacroAssembler::corrected_idivq(Register result, Register rs1, Register rs2, 2414 bool want_remainder) 2415 { 2416 // Full implementation of Java ldiv and lrem. The function 2417 // returns the (pc) offset of the div instruction - may be needed 2418 // for implicit exceptions. 2419 // 2420 // input : rs1: dividend 2421 // rs2: divisor 2422 // 2423 // result: either 2424 // quotient (= rs1 idiv rs2) 2425 // remainder (= rs1 irem rs2) 2426 2427 int idivq_offset = offset(); 2428 if (!want_remainder) { 2429 div(result, rs1, rs2); 2430 } else { 2431 rem(result, rs1, rs2); // result = rs1 % rs2; 2432 } 2433 return idivq_offset; 2434 } 2435 2436 // Look up the method for a megamorpic invkkeinterface call. 2437 // The target method is determined by <intf_klass, itable_index>. 2438 // The receiver klass is in recv_klass. 2439 // On success, the result will be in method_result, and execution falls through. 2440 // On failure, execution transfers to the given label. 2441 void MacroAssembler::lookup_interface_method(Register recv_klass, 2442 Register intf_klass, 2443 RegisterOrConstant itable_index, 2444 Register method_result, 2445 Register scan_tmp, 2446 Label& L_no_such_interface, 2447 bool return_method) { 2448 assert_different_registers(recv_klass, intf_klass, scan_tmp); 2449 assert_different_registers(method_result, intf_klass, scan_tmp); 2450 assert(recv_klass != method_result || !return_method, 2451 "recv_klass can be destroyed when mehtid isn't needed"); 2452 assert(itable_index.is_constant() || itable_index.as_register() == method_result, 2453 "caller must be same register for non-constant itable index as for method"); 2454 2455 // Compute start of first itableOffsetEntry (which is at the end of the vtable). 2456 int vtable_base = in_bytes(Klass::vtable_start_offset()); 2457 int itentry_off = in_bytes(itableMethodEntry::method_offset()); 2458 int scan_step = itableOffsetEntry::size() * wordSize; 2459 int vte_size = vtableEntry::size_in_bytes(); 2460 assert(vte_size == wordSize, "else adjust times_vte_scale"); 2461 2462 lwu(scan_tmp, Address(recv_klass, Klass::vtable_length_offset())); 2463 2464 // %%% Could store the aligned, prescaled offset in the klassoop. 2465 shadd(scan_tmp, scan_tmp, recv_klass, scan_tmp, 3); 2466 add(scan_tmp, scan_tmp, vtable_base); 2467 2468 if (return_method) { 2469 // Adjust recv_klass by scaled itable_index, so we can free itable_index. 2470 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below"); 2471 if (itable_index.is_register()) { 2472 slli(t0, itable_index.as_register(), 3); 2473 } else { 2474 mv(t0, itable_index.as_constant() << 3); 2475 } 2476 add(recv_klass, recv_klass, t0); 2477 if (itentry_off) { 2478 add(recv_klass, recv_klass, itentry_off); 2479 } 2480 } 2481 2482 Label search, found_method; 2483 2484 ld(method_result, Address(scan_tmp, itableOffsetEntry::interface_offset())); 2485 beq(intf_klass, method_result, found_method); 2486 bind(search); 2487 // Check that the previous entry is non-null. A null entry means that 2488 // the receiver class doesn't implement the interface, and wasn't the 2489 // same as when the caller was compiled. 2490 beqz(method_result, L_no_such_interface, /* is_far */ true); 2491 addi(scan_tmp, scan_tmp, scan_step); 2492 ld(method_result, Address(scan_tmp, itableOffsetEntry::interface_offset())); 2493 bne(intf_klass, method_result, search); 2494 2495 bind(found_method); 2496 2497 // Got a hit. 2498 if (return_method) { 2499 lwu(scan_tmp, Address(scan_tmp, itableOffsetEntry::offset_offset())); 2500 add(method_result, recv_klass, scan_tmp); 2501 ld(method_result, Address(method_result)); 2502 } 2503 } 2504 2505 // virtual method calling 2506 void MacroAssembler::lookup_virtual_method(Register recv_klass, 2507 RegisterOrConstant vtable_index, 2508 Register method_result) { 2509 const ByteSize base = Klass::vtable_start_offset(); 2510 assert(vtableEntry::size() * wordSize == 8, 2511 "adjust the scaling in the code below"); 2512 int vtable_offset_in_bytes = in_bytes(base + vtableEntry::method_offset()); 2513 2514 if (vtable_index.is_register()) { 2515 shadd(method_result, vtable_index.as_register(), recv_klass, method_result, LogBytesPerWord); 2516 ld(method_result, Address(method_result, vtable_offset_in_bytes)); 2517 } else { 2518 vtable_offset_in_bytes += vtable_index.as_constant() * wordSize; 2519 ld(method_result, form_address(method_result, recv_klass, vtable_offset_in_bytes)); 2520 } 2521 } 2522 2523 void MacroAssembler::membar(uint32_t order_constraint) { 2524 address prev = pc() - NativeMembar::instruction_size; 2525 address last = code()->last_insn(); 2526 2527 if (last != nullptr && nativeInstruction_at(last)->is_membar() && prev == last) { 2528 NativeMembar *bar = NativeMembar_at(prev); 2529 // We are merging two memory barrier instructions. On RISCV we 2530 // can do this simply by ORing them together. 2531 bar->set_kind(bar->get_kind() | order_constraint); 2532 BLOCK_COMMENT("merged membar"); 2533 } else { 2534 code()->set_last_insn(pc()); 2535 2536 uint32_t predecessor = 0; 2537 uint32_t successor = 0; 2538 2539 membar_mask_to_pred_succ(order_constraint, predecessor, successor); 2540 fence(predecessor, successor); 2541 } 2542 } 2543 2544 // Form an address from base + offset in Rd. Rd my or may not 2545 // actually be used: you must use the Address that is returned. It 2546 // is up to you to ensure that the shift provided matches the size 2547 // of your data. 2548 Address MacroAssembler::form_address(Register Rd, Register base, int64_t byte_offset) { 2549 if (is_simm12(byte_offset)) { // 12: imm in range 2^12 2550 return Address(base, byte_offset); 2551 } 2552 2553 assert_different_registers(Rd, base, noreg); 2554 2555 // Do it the hard way 2556 mv(Rd, byte_offset); 2557 add(Rd, base, Rd); 2558 return Address(Rd); 2559 } 2560 2561 void MacroAssembler::check_klass_subtype(Register sub_klass, 2562 Register super_klass, 2563 Register tmp_reg, 2564 Label& L_success) { 2565 Label L_failure; 2566 check_klass_subtype_fast_path(sub_klass, super_klass, tmp_reg, &L_success, &L_failure, nullptr); 2567 check_klass_subtype_slow_path(sub_klass, super_klass, tmp_reg, noreg, &L_success, nullptr); 2568 bind(L_failure); 2569 } 2570 2571 void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod) { 2572 ld(t0, Address(xthread, JavaThread::polling_word_offset())); 2573 if (acquire) { 2574 membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore); 2575 } 2576 if (at_return) { 2577 bgtu(in_nmethod ? sp : fp, t0, slow_path, /* is_far */ true); 2578 } else { 2579 test_bit(t0, t0, exact_log2(SafepointMechanism::poll_bit())); 2580 bnez(t0, slow_path, true /* is_far */); 2581 } 2582 } 2583 2584 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp, 2585 Label &succeed, Label *fail) { 2586 assert_different_registers(addr, tmp); 2587 assert_different_registers(newv, tmp); 2588 assert_different_registers(oldv, tmp); 2589 2590 // oldv holds comparison value 2591 // newv holds value to write in exchange 2592 // addr identifies memory word to compare against/update 2593 Label retry_load, nope; 2594 bind(retry_load); 2595 // Load reserved from the memory location 2596 lr_d(tmp, addr, Assembler::aqrl); 2597 // Fail and exit if it is not what we expect 2598 bne(tmp, oldv, nope); 2599 // If the store conditional succeeds, tmp will be zero 2600 sc_d(tmp, newv, addr, Assembler::rl); 2601 beqz(tmp, succeed); 2602 // Retry only when the store conditional failed 2603 j(retry_load); 2604 2605 bind(nope); 2606 membar(AnyAny); 2607 mv(oldv, tmp); 2608 if (fail != nullptr) { 2609 j(*fail); 2610 } 2611 } 2612 2613 void MacroAssembler::cmpxchg_obj_header(Register oldv, Register newv, Register obj, Register tmp, 2614 Label &succeed, Label *fail) { 2615 assert(oopDesc::mark_offset_in_bytes() == 0, "assumption"); 2616 cmpxchgptr(oldv, newv, obj, tmp, succeed, fail); 2617 } 2618 2619 void MacroAssembler::load_reserved(Register addr, 2620 enum operand_size size, 2621 Assembler::Aqrl acquire) { 2622 switch (size) { 2623 case int64: 2624 lr_d(t0, addr, acquire); 2625 break; 2626 case int32: 2627 lr_w(t0, addr, acquire); 2628 break; 2629 case uint32: 2630 lr_w(t0, addr, acquire); 2631 zero_extend(t0, t0, 32); 2632 break; 2633 default: 2634 ShouldNotReachHere(); 2635 } 2636 } 2637 2638 void MacroAssembler::store_conditional(Register addr, 2639 Register new_val, 2640 enum operand_size size, 2641 Assembler::Aqrl release) { 2642 switch (size) { 2643 case int64: 2644 sc_d(t0, new_val, addr, release); 2645 break; 2646 case int32: 2647 case uint32: 2648 sc_w(t0, new_val, addr, release); 2649 break; 2650 default: 2651 ShouldNotReachHere(); 2652 } 2653 } 2654 2655 2656 void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected, 2657 Register new_val, 2658 enum operand_size size, 2659 Register tmp1, Register tmp2, Register tmp3) { 2660 assert(size == int8 || size == int16, "unsupported operand size"); 2661 2662 Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3; 2663 2664 andi(shift, addr, 3); 2665 slli(shift, shift, 3); 2666 2667 andi(aligned_addr, addr, ~3); 2668 2669 if (size == int8) { 2670 mv(mask, 0xff); 2671 } else { 2672 // size == int16 case 2673 mv(mask, -1); 2674 zero_extend(mask, mask, 16); 2675 } 2676 sll(mask, mask, shift); 2677 2678 xori(not_mask, mask, -1); 2679 2680 sll(expected, expected, shift); 2681 andr(expected, expected, mask); 2682 2683 sll(new_val, new_val, shift); 2684 andr(new_val, new_val, mask); 2685 } 2686 2687 // cmpxchg_narrow_value will kill t0, t1, expected, new_val and tmps. 2688 // It's designed to implement compare and swap byte/boolean/char/short by lr.w/sc.w, 2689 // which are forced to work with 4-byte aligned address. 2690 void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected, 2691 Register new_val, 2692 enum operand_size size, 2693 Assembler::Aqrl acquire, Assembler::Aqrl release, 2694 Register result, bool result_as_bool, 2695 Register tmp1, Register tmp2, Register tmp3) { 2696 Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0; 2697 assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp); 2698 cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3); 2699 2700 Label retry, fail, done; 2701 2702 bind(retry); 2703 lr_w(old, aligned_addr, acquire); 2704 andr(tmp, old, mask); 2705 bne(tmp, expected, fail); 2706 2707 andr(tmp, old, not_mask); 2708 orr(tmp, tmp, new_val); 2709 sc_w(tmp, tmp, aligned_addr, release); 2710 bnez(tmp, retry); 2711 2712 if (result_as_bool) { 2713 mv(result, 1); 2714 j(done); 2715 2716 bind(fail); 2717 mv(result, zr); 2718 2719 bind(done); 2720 } else { 2721 andr(tmp, old, mask); 2722 2723 bind(fail); 2724 srl(result, tmp, shift); 2725 2726 if (size == int8) { 2727 sign_extend(result, result, 8); 2728 } else { 2729 // size == int16 case 2730 sign_extend(result, result, 16); 2731 } 2732 } 2733 } 2734 2735 // weak_cmpxchg_narrow_value is a weak version of cmpxchg_narrow_value, to implement 2736 // the weak CAS stuff. The major difference is that it just failed when store conditional 2737 // failed. 2738 void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected, 2739 Register new_val, 2740 enum operand_size size, 2741 Assembler::Aqrl acquire, Assembler::Aqrl release, 2742 Register result, 2743 Register tmp1, Register tmp2, Register tmp3) { 2744 Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0; 2745 assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp); 2746 cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3); 2747 2748 Label fail, done; 2749 2750 lr_w(old, aligned_addr, acquire); 2751 andr(tmp, old, mask); 2752 bne(tmp, expected, fail); 2753 2754 andr(tmp, old, not_mask); 2755 orr(tmp, tmp, new_val); 2756 sc_w(tmp, tmp, aligned_addr, release); 2757 bnez(tmp, fail); 2758 2759 // Success 2760 mv(result, 1); 2761 j(done); 2762 2763 // Fail 2764 bind(fail); 2765 mv(result, zr); 2766 2767 bind(done); 2768 } 2769 2770 void MacroAssembler::cmpxchg(Register addr, Register expected, 2771 Register new_val, 2772 enum operand_size size, 2773 Assembler::Aqrl acquire, Assembler::Aqrl release, 2774 Register result, bool result_as_bool) { 2775 assert(size != int8 && size != int16, "unsupported operand size"); 2776 assert_different_registers(addr, t0); 2777 assert_different_registers(expected, t0); 2778 assert_different_registers(new_val, t0); 2779 2780 Label retry_load, done, ne_done; 2781 bind(retry_load); 2782 load_reserved(addr, size, acquire); 2783 bne(t0, expected, ne_done); 2784 store_conditional(addr, new_val, size, release); 2785 bnez(t0, retry_load); 2786 2787 // equal, succeed 2788 if (result_as_bool) { 2789 mv(result, 1); 2790 } else { 2791 mv(result, expected); 2792 } 2793 j(done); 2794 2795 // not equal, failed 2796 bind(ne_done); 2797 if (result_as_bool) { 2798 mv(result, zr); 2799 } else { 2800 mv(result, t0); 2801 } 2802 2803 bind(done); 2804 } 2805 2806 void MacroAssembler::cmpxchg_weak(Register addr, Register expected, 2807 Register new_val, 2808 enum operand_size size, 2809 Assembler::Aqrl acquire, Assembler::Aqrl release, 2810 Register result) { 2811 assert_different_registers(addr, t0); 2812 assert_different_registers(expected, t0); 2813 assert_different_registers(new_val, t0); 2814 2815 Label fail, done; 2816 load_reserved(addr, size, acquire); 2817 bne(t0, expected, fail); 2818 store_conditional(addr, new_val, size, release); 2819 bnez(t0, fail); 2820 2821 // Success 2822 mv(result, 1); 2823 j(done); 2824 2825 // Fail 2826 bind(fail); 2827 mv(result, zr); 2828 2829 bind(done); 2830 } 2831 2832 #define ATOMIC_OP(NAME, AOP, ACQUIRE, RELEASE) \ 2833 void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \ 2834 prev = prev->is_valid() ? prev : zr; \ 2835 if (incr.is_register()) { \ 2836 AOP(prev, addr, incr.as_register(), (Assembler::Aqrl)(ACQUIRE | RELEASE)); \ 2837 } else { \ 2838 mv(t0, incr.as_constant()); \ 2839 AOP(prev, addr, t0, (Assembler::Aqrl)(ACQUIRE | RELEASE)); \ 2840 } \ 2841 return; \ 2842 } 2843 2844 ATOMIC_OP(add, amoadd_d, Assembler::relaxed, Assembler::relaxed) 2845 ATOMIC_OP(addw, amoadd_w, Assembler::relaxed, Assembler::relaxed) 2846 ATOMIC_OP(addal, amoadd_d, Assembler::aq, Assembler::rl) 2847 ATOMIC_OP(addalw, amoadd_w, Assembler::aq, Assembler::rl) 2848 2849 #undef ATOMIC_OP 2850 2851 #define ATOMIC_XCHG(OP, AOP, ACQUIRE, RELEASE) \ 2852 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) { \ 2853 prev = prev->is_valid() ? prev : zr; \ 2854 AOP(prev, addr, newv, (Assembler::Aqrl)(ACQUIRE | RELEASE)); \ 2855 return; \ 2856 } 2857 2858 ATOMIC_XCHG(xchg, amoswap_d, Assembler::relaxed, Assembler::relaxed) 2859 ATOMIC_XCHG(xchgw, amoswap_w, Assembler::relaxed, Assembler::relaxed) 2860 ATOMIC_XCHG(xchgal, amoswap_d, Assembler::aq, Assembler::rl) 2861 ATOMIC_XCHG(xchgalw, amoswap_w, Assembler::aq, Assembler::rl) 2862 2863 #undef ATOMIC_XCHG 2864 2865 #define ATOMIC_XCHGU(OP1, OP2) \ 2866 void MacroAssembler::atomic_##OP1(Register prev, Register newv, Register addr) { \ 2867 atomic_##OP2(prev, newv, addr); \ 2868 zero_extend(prev, prev, 32); \ 2869 return; \ 2870 } 2871 2872 ATOMIC_XCHGU(xchgwu, xchgw) 2873 ATOMIC_XCHGU(xchgalwu, xchgalw) 2874 2875 #undef ATOMIC_XCHGU 2876 2877 void MacroAssembler::far_jump(Address entry, Register tmp) { 2878 assert(ReservedCodeCacheSize < 4*G, "branch out of range"); 2879 assert(CodeCache::find_blob(entry.target()) != nullptr, 2880 "destination of far call not found in code cache"); 2881 assert(entry.rspec().type() == relocInfo::external_word_type 2882 || entry.rspec().type() == relocInfo::runtime_call_type 2883 || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type"); 2884 IncompressibleRegion ir(this); // Fixed length: see MacroAssembler::far_branch_size() 2885 if (far_branches()) { 2886 // We can use auipc + jalr here because we know that the total size of 2887 // the code cache cannot exceed 2Gb. 2888 relocate(entry.rspec(), [&] { 2889 int32_t offset; 2890 la_patchable(tmp, entry, offset); 2891 jalr(x0, tmp, offset); 2892 }); 2893 } else { 2894 j(entry); 2895 } 2896 } 2897 2898 void MacroAssembler::far_call(Address entry, Register tmp) { 2899 assert(ReservedCodeCacheSize < 4*G, "branch out of range"); 2900 assert(CodeCache::find_blob(entry.target()) != nullptr, 2901 "destination of far call not found in code cache"); 2902 assert(entry.rspec().type() == relocInfo::external_word_type 2903 || entry.rspec().type() == relocInfo::runtime_call_type 2904 || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type"); 2905 IncompressibleRegion ir(this); // Fixed length: see MacroAssembler::far_branch_size() 2906 if (far_branches()) { 2907 // We can use auipc + jalr here because we know that the total size of 2908 // the code cache cannot exceed 2Gb. 2909 relocate(entry.rspec(), [&] { 2910 int32_t offset; 2911 la_patchable(tmp, entry, offset); 2912 jalr(x1, tmp, offset); // link 2913 }); 2914 } else { 2915 jal(entry); // link 2916 } 2917 } 2918 2919 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass, 2920 Register super_klass, 2921 Register tmp_reg, 2922 Label* L_success, 2923 Label* L_failure, 2924 Label* L_slow_path, 2925 Register super_check_offset) { 2926 assert_different_registers(sub_klass, super_klass, tmp_reg); 2927 bool must_load_sco = (super_check_offset == noreg); 2928 if (must_load_sco) { 2929 assert(tmp_reg != noreg, "supply either a temp or a register offset"); 2930 } else { 2931 assert_different_registers(sub_klass, super_klass, super_check_offset); 2932 } 2933 2934 Label L_fallthrough; 2935 int label_nulls = 0; 2936 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 2937 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 2938 if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; } 2939 assert(label_nulls <= 1, "at most one null in batch"); 2940 2941 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 2942 int sco_offset = in_bytes(Klass::super_check_offset_offset()); 2943 Address super_check_offset_addr(super_klass, sco_offset); 2944 2945 // Hacked jmp, which may only be used just before L_fallthrough. 2946 #define final_jmp(label) \ 2947 if (&(label) == &L_fallthrough) { /*do nothing*/ } \ 2948 else j(label) /*omit semi*/ 2949 2950 // If the pointers are equal, we are done (e.g., String[] elements). 2951 // This self-check enables sharing of secondary supertype arrays among 2952 // non-primary types such as array-of-interface. Otherwise, each such 2953 // type would need its own customized SSA. 2954 // We move this check to the front of the fast path because many 2955 // type checks are in fact trivially successful in this manner, 2956 // so we get a nicely predicted branch right at the start of the check. 2957 beq(sub_klass, super_klass, *L_success); 2958 2959 // Check the supertype display: 2960 if (must_load_sco) { 2961 lwu(tmp_reg, super_check_offset_addr); 2962 super_check_offset = tmp_reg; 2963 } 2964 add(t0, sub_klass, super_check_offset); 2965 Address super_check_addr(t0); 2966 ld(t0, super_check_addr); // load displayed supertype 2967 2968 // This check has worked decisively for primary supers. 2969 // Secondary supers are sought in the super_cache ('super_cache_addr'). 2970 // (Secondary supers are interfaces and very deeply nested subtypes.) 2971 // This works in the same check above because of a tricky aliasing 2972 // between the super_Cache and the primary super display elements. 2973 // (The 'super_check_addr' can address either, as the case requires.) 2974 // Note that the cache is updated below if it does not help us find 2975 // what we need immediately. 2976 // So if it was a primary super, we can just fail immediately. 2977 // Otherwise, it's the slow path for us (no success at this point). 2978 2979 beq(super_klass, t0, *L_success); 2980 mv(t1, sc_offset); 2981 if (L_failure == &L_fallthrough) { 2982 beq(super_check_offset, t1, *L_slow_path); 2983 } else { 2984 bne(super_check_offset, t1, *L_failure, /* is_far */ true); 2985 final_jmp(*L_slow_path); 2986 } 2987 2988 bind(L_fallthrough); 2989 2990 #undef final_jmp 2991 } 2992 2993 // Scans count pointer sized words at [addr] for occurrence of value, 2994 // generic 2995 void MacroAssembler::repne_scan(Register addr, Register value, Register count, 2996 Register tmp) { 2997 Label Lloop, Lexit; 2998 beqz(count, Lexit); 2999 bind(Lloop); 3000 ld(tmp, addr); 3001 beq(value, tmp, Lexit); 3002 add(addr, addr, wordSize); 3003 sub(count, count, 1); 3004 bnez(count, Lloop); 3005 bind(Lexit); 3006 } 3007 3008 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass, 3009 Register super_klass, 3010 Register tmp1_reg, 3011 Register tmp2_reg, 3012 Label* L_success, 3013 Label* L_failure) { 3014 assert_different_registers(sub_klass, super_klass, tmp1_reg); 3015 if (tmp2_reg != noreg) { 3016 assert_different_registers(sub_klass, super_klass, tmp1_reg, tmp2_reg, t0); 3017 } 3018 #define IS_A_TEMP(reg) ((reg) == tmp1_reg || (reg) == tmp2_reg) 3019 3020 Label L_fallthrough; 3021 int label_nulls = 0; 3022 if (L_success == nullptr) { L_success = &L_fallthrough; label_nulls++; } 3023 if (L_failure == nullptr) { L_failure = &L_fallthrough; label_nulls++; } 3024 3025 assert(label_nulls <= 1, "at most one null in the batch"); 3026 3027 // A couple of useful fields in sub_klass: 3028 int ss_offset = in_bytes(Klass::secondary_supers_offset()); 3029 int sc_offset = in_bytes(Klass::secondary_super_cache_offset()); 3030 Address secondary_supers_addr(sub_klass, ss_offset); 3031 Address super_cache_addr( sub_klass, sc_offset); 3032 3033 BLOCK_COMMENT("check_klass_subtype_slow_path"); 3034 3035 // Do a linear scan of the secondary super-klass chain. 3036 // This code is rarely used, so simplicity is a virtue here. 3037 // The repne_scan instruction uses fixed registers, which we must spill. 3038 // Don't worry too much about pre-existing connections with the input regs. 3039 3040 assert(sub_klass != x10, "killed reg"); // killed by mv(x10, super) 3041 assert(sub_klass != x12, "killed reg"); // killed by la(x12, &pst_counter) 3042 3043 RegSet pushed_registers; 3044 if (!IS_A_TEMP(x12)) { 3045 pushed_registers += x12; 3046 } 3047 if (!IS_A_TEMP(x15)) { 3048 pushed_registers += x15; 3049 } 3050 3051 if (super_klass != x10) { 3052 if (!IS_A_TEMP(x10)) { 3053 pushed_registers += x10; 3054 } 3055 } 3056 3057 push_reg(pushed_registers, sp); 3058 3059 // Get super_klass value into x10 (even if it was in x15 or x12) 3060 mv(x10, super_klass); 3061 3062 #ifndef PRODUCT 3063 mv(t1, (address)&SharedRuntime::_partial_subtype_ctr); 3064 Address pst_counter_addr(t1); 3065 ld(t0, pst_counter_addr); 3066 add(t0, t0, 1); 3067 sd(t0, pst_counter_addr); 3068 #endif // PRODUCT 3069 3070 // We will consult the secondary-super array. 3071 ld(x15, secondary_supers_addr); 3072 // Load the array length. 3073 lwu(x12, Address(x15, Array<Klass*>::length_offset_in_bytes())); 3074 // Skip to start of data. 3075 add(x15, x15, Array<Klass*>::base_offset_in_bytes()); 3076 3077 // Set t0 to an obvious invalid value, falling through by default 3078 mv(t0, -1); 3079 // Scan X12 words at [X15] for an occurrence of X10. 3080 repne_scan(x15, x10, x12, t0); 3081 3082 // pop will restore x10, so we should use a temp register to keep its value 3083 mv(t1, x10); 3084 3085 // Unspill the temp registers: 3086 pop_reg(pushed_registers, sp); 3087 3088 bne(t1, t0, *L_failure); 3089 3090 // Success. Cache the super we found an proceed in triumph. 3091 sd(super_klass, super_cache_addr); 3092 3093 if (L_success != &L_fallthrough) { 3094 j(*L_success); 3095 } 3096 3097 #undef IS_A_TEMP 3098 3099 bind(L_fallthrough); 3100 } 3101 3102 // Defines obj, preserves var_size_in_bytes, okay for tmp2 == var_size_in_bytes. 3103 void MacroAssembler::tlab_allocate(Register obj, 3104 Register var_size_in_bytes, 3105 int con_size_in_bytes, 3106 Register tmp1, 3107 Register tmp2, 3108 Label& slow_case, 3109 bool is_far) { 3110 BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler(); 3111 bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, tmp1, tmp2, slow_case, is_far); 3112 } 3113 3114 // get_thread() can be called anywhere inside generated code so we 3115 // need to save whatever non-callee save context might get clobbered 3116 // by the call to Thread::current() or, indeed, the call setup code. 3117 void MacroAssembler::get_thread(Register thread) { 3118 // save all call-clobbered regs except thread 3119 RegSet saved_regs = RegSet::range(x5, x7) + RegSet::range(x10, x17) + 3120 RegSet::range(x28, x31) + ra - thread; 3121 push_reg(saved_regs, sp); 3122 3123 mv(ra, CAST_FROM_FN_PTR(address, Thread::current)); 3124 jalr(ra); 3125 if (thread != c_rarg0) { 3126 mv(thread, c_rarg0); 3127 } 3128 3129 // restore pushed registers 3130 pop_reg(saved_regs, sp); 3131 } 3132 3133 void MacroAssembler::load_byte_map_base(Register reg) { 3134 CardTable::CardValue* byte_map_base = 3135 ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base(); 3136 mv(reg, (uint64_t)byte_map_base); 3137 } 3138 3139 void MacroAssembler::la_patchable(Register reg1, const Address &dest, int32_t &offset) { 3140 unsigned long low_address = (uintptr_t)CodeCache::low_bound(); 3141 unsigned long high_address = (uintptr_t)CodeCache::high_bound(); 3142 unsigned long dest_address = (uintptr_t)dest.target(); 3143 long offset_low = dest_address - low_address; 3144 long offset_high = dest_address - high_address; 3145 3146 assert(dest.getMode() == Address::literal, "la_patchable must be applied to a literal address"); 3147 assert((uintptr_t)dest.target() < (1ull << 48), "bad address"); 3148 3149 // RISC-V doesn't compute a page-aligned address, in order to partially 3150 // compensate for the use of *signed* offsets in its base+disp12 3151 // addressing mode (RISC-V's PC-relative reach remains asymmetric 3152 // [-(2G + 2K), 2G - 2K). 3153 if (offset_high >= -((1L << 31) + (1L << 11)) && offset_low < (1L << 31) - (1L << 11)) { 3154 int64_t distance = dest.target() - pc(); 3155 auipc(reg1, (int32_t)distance + 0x800); 3156 offset = ((int32_t)distance << 20) >> 20; 3157 } else { 3158 movptr(reg1, dest.target(), offset); 3159 } 3160 } 3161 3162 void MacroAssembler::build_frame(int framesize) { 3163 assert(framesize >= 2, "framesize must include space for FP/RA"); 3164 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment"); 3165 sub(sp, sp, framesize); 3166 sd(fp, Address(sp, framesize - 2 * wordSize)); 3167 sd(ra, Address(sp, framesize - wordSize)); 3168 if (PreserveFramePointer) { add(fp, sp, framesize); } 3169 } 3170 3171 void MacroAssembler::remove_frame(int framesize) { 3172 assert(framesize >= 2, "framesize must include space for FP/RA"); 3173 assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment"); 3174 ld(fp, Address(sp, framesize - 2 * wordSize)); 3175 ld(ra, Address(sp, framesize - wordSize)); 3176 add(sp, sp, framesize); 3177 } 3178 3179 void MacroAssembler::reserved_stack_check() { 3180 // testing if reserved zone needs to be enabled 3181 Label no_reserved_zone_enabling; 3182 3183 ld(t0, Address(xthread, JavaThread::reserved_stack_activation_offset())); 3184 bltu(sp, t0, no_reserved_zone_enabling); 3185 3186 enter(); // RA and FP are live. 3187 mv(c_rarg0, xthread); 3188 RuntimeAddress target(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone)); 3189 relocate(target.rspec(), [&] { 3190 int32_t offset; 3191 la_patchable(t0, target, offset); 3192 jalr(x1, t0, offset); 3193 }); 3194 leave(); 3195 3196 // We have already removed our own frame. 3197 // throw_delayed_StackOverflowError will think that it's been 3198 // called by our caller. 3199 target = RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry()); 3200 relocate(target.rspec(), [&] { 3201 int32_t offset; 3202 la_patchable(t0, target, offset); 3203 jalr(x0, t0, offset); 3204 }); 3205 should_not_reach_here(); 3206 3207 bind(no_reserved_zone_enabling); 3208 } 3209 3210 // Move the address of the polling page into dest. 3211 void MacroAssembler::get_polling_page(Register dest, relocInfo::relocType rtype) { 3212 ld(dest, Address(xthread, JavaThread::polling_page_offset())); 3213 } 3214 3215 // Read the polling page. The address of the polling page must 3216 // already be in r. 3217 void MacroAssembler::read_polling_page(Register r, int32_t offset, relocInfo::relocType rtype) { 3218 relocate(rtype, [&] { 3219 lwu(zr, Address(r, offset)); 3220 }); 3221 } 3222 3223 void MacroAssembler::set_narrow_oop(Register dst, jobject obj) { 3224 #ifdef ASSERT 3225 { 3226 ThreadInVMfromUnknown tiv; 3227 assert (UseCompressedOops, "should only be used for compressed oops"); 3228 assert (Universe::heap() != nullptr, "java heap should be initialized"); 3229 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 3230 assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop"); 3231 } 3232 #endif 3233 int oop_index = oop_recorder()->find_index(obj); 3234 relocate(oop_Relocation::spec(oop_index), [&] { 3235 li32(dst, 0xDEADBEEF); 3236 }); 3237 zero_extend(dst, dst, 32); 3238 } 3239 3240 void MacroAssembler::set_narrow_klass(Register dst, Klass* k) { 3241 assert (UseCompressedClassPointers, "should only be used for compressed headers"); 3242 assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder"); 3243 int index = oop_recorder()->find_index(k); 3244 assert(!Universe::heap()->is_in(k), "should not be an oop"); 3245 3246 narrowKlass nk = CompressedKlassPointers::encode(k); 3247 relocate(metadata_Relocation::spec(index), [&] { 3248 li32(dst, nk); 3249 }); 3250 zero_extend(dst, dst, 32); 3251 } 3252 3253 // Maybe emit a call via a trampoline. If the code cache is small 3254 // trampolines won't be emitted. 3255 address MacroAssembler::trampoline_call(Address entry) { 3256 assert(entry.rspec().type() == relocInfo::runtime_call_type || 3257 entry.rspec().type() == relocInfo::opt_virtual_call_type || 3258 entry.rspec().type() == relocInfo::static_call_type || 3259 entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type"); 3260 3261 address target = entry.target(); 3262 3263 // We need a trampoline if branches are far. 3264 if (far_branches()) { 3265 if (!in_scratch_emit_size()) { 3266 if (entry.rspec().type() == relocInfo::runtime_call_type) { 3267 assert(CodeBuffer::supports_shared_stubs(), "must support shared stubs"); 3268 code()->share_trampoline_for(entry.target(), offset()); 3269 } else { 3270 address stub = emit_trampoline_stub(offset(), target); 3271 if (stub == nullptr) { 3272 postcond(pc() == badAddress); 3273 return nullptr; // CodeCache is full 3274 } 3275 } 3276 } 3277 target = pc(); 3278 } 3279 3280 address call_pc = pc(); 3281 #ifdef ASSERT 3282 if (entry.rspec().type() != relocInfo::runtime_call_type) { 3283 assert_alignment(call_pc); 3284 } 3285 #endif 3286 relocate(entry.rspec(), [&] { 3287 jal(target); 3288 }); 3289 3290 postcond(pc() != badAddress); 3291 return call_pc; 3292 } 3293 3294 address MacroAssembler::ic_call(address entry, jint method_index) { 3295 RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index); 3296 IncompressibleRegion ir(this); // relocations 3297 movptr(t1, (address)Universe::non_oop_word()); 3298 assert_cond(entry != nullptr); 3299 return trampoline_call(Address(entry, rh)); 3300 } 3301 3302 // Emit a trampoline stub for a call to a target which is too far away. 3303 // 3304 // code sequences: 3305 // 3306 // call-site: 3307 // branch-and-link to <destination> or <trampoline stub> 3308 // 3309 // Related trampoline stub for this call site in the stub section: 3310 // load the call target from the constant pool 3311 // branch (RA still points to the call site above) 3312 3313 address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset, 3314 address dest) { 3315 // Max stub size: alignment nop, TrampolineStub. 3316 address stub = start_a_stub(max_trampoline_stub_size()); 3317 if (stub == nullptr) { 3318 return nullptr; // CodeBuffer::expand failed 3319 } 3320 3321 // We are always 4-byte aligned here. 3322 assert_alignment(pc()); 3323 3324 // Create a trampoline stub relocation which relates this trampoline stub 3325 // with the call instruction at insts_call_instruction_offset in the 3326 // instructions code-section. 3327 3328 // Make sure the address of destination 8-byte aligned after 3 instructions. 3329 align(wordSize, NativeCallTrampolineStub::data_offset); 3330 3331 RelocationHolder rh = trampoline_stub_Relocation::spec(code()->insts()->start() + 3332 insts_call_instruction_offset); 3333 const int stub_start_offset = offset(); 3334 relocate(rh, [&] { 3335 // Now, create the trampoline stub's code: 3336 // - load the call 3337 // - call 3338 Label target; 3339 ld(t0, target); // auipc + ld 3340 jr(t0); // jalr 3341 bind(target); 3342 assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset, 3343 "should be"); 3344 assert(offset() % wordSize == 0, "bad alignment"); 3345 emit_int64((int64_t)dest); 3346 }); 3347 3348 const address stub_start_addr = addr_at(stub_start_offset); 3349 3350 assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline"); 3351 3352 end_a_stub(); 3353 return stub_start_addr; 3354 } 3355 3356 int MacroAssembler::max_trampoline_stub_size() { 3357 // Max stub size: alignment nop, TrampolineStub. 3358 return NativeInstruction::instruction_size + NativeCallTrampolineStub::instruction_size; 3359 } 3360 3361 int MacroAssembler::static_call_stub_size() { 3362 // (lui, addi, slli, addi, slli, addi) + (lui, addi, slli, addi, slli) + jalr 3363 return 12 * NativeInstruction::instruction_size; 3364 } 3365 3366 Address MacroAssembler::add_memory_helper(const Address dst, Register tmp) { 3367 switch (dst.getMode()) { 3368 case Address::base_plus_offset: 3369 // This is the expected mode, although we allow all the other 3370 // forms below. 3371 return form_address(tmp, dst.base(), dst.offset()); 3372 default: 3373 la(tmp, dst); 3374 return Address(tmp); 3375 } 3376 } 3377 3378 void MacroAssembler::increment(const Address dst, int64_t value, Register tmp1, Register tmp2) { 3379 assert(((dst.getMode() == Address::base_plus_offset && 3380 is_simm12(dst.offset())) || is_simm12(value)), 3381 "invalid value and address mode combination"); 3382 Address adr = add_memory_helper(dst, tmp2); 3383 assert(!adr.uses(tmp1), "invalid dst for address increment"); 3384 ld(tmp1, adr); 3385 add(tmp1, tmp1, value, tmp2); 3386 sd(tmp1, adr); 3387 } 3388 3389 void MacroAssembler::incrementw(const Address dst, int32_t value, Register tmp1, Register tmp2) { 3390 assert(((dst.getMode() == Address::base_plus_offset && 3391 is_simm12(dst.offset())) || is_simm12(value)), 3392 "invalid value and address mode combination"); 3393 Address adr = add_memory_helper(dst, tmp2); 3394 assert(!adr.uses(tmp1), "invalid dst for address increment"); 3395 lwu(tmp1, adr); 3396 addw(tmp1, tmp1, value, tmp2); 3397 sw(tmp1, adr); 3398 } 3399 3400 void MacroAssembler::decrement(const Address dst, int64_t value, Register tmp1, Register tmp2) { 3401 assert(((dst.getMode() == Address::base_plus_offset && 3402 is_simm12(dst.offset())) || is_simm12(value)), 3403 "invalid value and address mode combination"); 3404 Address adr = add_memory_helper(dst, tmp2); 3405 assert(!adr.uses(tmp1), "invalid dst for address decrement"); 3406 ld(tmp1, adr); 3407 sub(tmp1, tmp1, value, tmp2); 3408 sd(tmp1, adr); 3409 } 3410 3411 void MacroAssembler::decrementw(const Address dst, int32_t value, Register tmp1, Register tmp2) { 3412 assert(((dst.getMode() == Address::base_plus_offset && 3413 is_simm12(dst.offset())) || is_simm12(value)), 3414 "invalid value and address mode combination"); 3415 Address adr = add_memory_helper(dst, tmp2); 3416 assert(!adr.uses(tmp1), "invalid dst for address decrement"); 3417 lwu(tmp1, adr); 3418 subw(tmp1, tmp1, value, tmp2); 3419 sw(tmp1, adr); 3420 } 3421 3422 void MacroAssembler::cmpptr(Register src1, Address src2, Label& equal) { 3423 assert_different_registers(src1, t0); 3424 relocate(src2.rspec(), [&] { 3425 int32_t offset; 3426 la_patchable(t0, src2, offset); 3427 ld(t0, Address(t0, offset)); 3428 }); 3429 beq(src1, t0, equal); 3430 } 3431 3432 void MacroAssembler::load_method_holder_cld(Register result, Register method) { 3433 load_method_holder(result, method); 3434 ld(result, Address(result, InstanceKlass::class_loader_data_offset())); 3435 } 3436 3437 void MacroAssembler::load_method_holder(Register holder, Register method) { 3438 ld(holder, Address(method, Method::const_offset())); // ConstMethod* 3439 ld(holder, Address(holder, ConstMethod::constants_offset())); // ConstantPool* 3440 ld(holder, Address(holder, ConstantPool::pool_holder_offset())); // InstanceKlass* 3441 } 3442 3443 // string indexof 3444 // compute index by trailing zeros 3445 void MacroAssembler::compute_index(Register haystack, Register trailing_zeros, 3446 Register match_mask, Register result, 3447 Register ch2, Register tmp, 3448 bool haystack_isL) { 3449 int haystack_chr_shift = haystack_isL ? 0 : 1; 3450 srl(match_mask, match_mask, trailing_zeros); 3451 srli(match_mask, match_mask, 1); 3452 srli(tmp, trailing_zeros, LogBitsPerByte); 3453 if (!haystack_isL) andi(tmp, tmp, 0xE); 3454 add(haystack, haystack, tmp); 3455 ld(ch2, Address(haystack)); 3456 if (!haystack_isL) srli(tmp, tmp, haystack_chr_shift); 3457 add(result, result, tmp); 3458 } 3459 3460 // string indexof 3461 // Find pattern element in src, compute match mask, 3462 // only the first occurrence of 0x80/0x8000 at low bits is the valid match index 3463 // match mask patterns and corresponding indices would be like: 3464 // - 0x8080808080808080 (Latin1) 3465 // - 7 6 5 4 3 2 1 0 (match index) 3466 // - 0x8000800080008000 (UTF16) 3467 // - 3 2 1 0 (match index) 3468 void MacroAssembler::compute_match_mask(Register src, Register pattern, Register match_mask, 3469 Register mask1, Register mask2) { 3470 xorr(src, pattern, src); 3471 sub(match_mask, src, mask1); 3472 orr(src, src, mask2); 3473 notr(src, src); 3474 andr(match_mask, match_mask, src); 3475 } 3476 3477 #ifdef COMPILER2 3478 // Code for BigInteger::mulAdd intrinsic 3479 // out = x10 3480 // in = x11 3481 // offset = x12 (already out.length-offset) 3482 // len = x13 3483 // k = x14 3484 // tmp = x28 3485 // 3486 // pseudo code from java implementation: 3487 // long kLong = k & LONG_MASK; 3488 // carry = 0; 3489 // offset = out.length-offset - 1; 3490 // for (int j = len - 1; j >= 0; j--) { 3491 // product = (in[j] & LONG_MASK) * kLong + (out[offset] & LONG_MASK) + carry; 3492 // out[offset--] = (int)product; 3493 // carry = product >>> 32; 3494 // } 3495 // return (int)carry; 3496 void MacroAssembler::mul_add(Register out, Register in, Register offset, 3497 Register len, Register k, Register tmp) { 3498 Label L_tail_loop, L_unroll, L_end; 3499 mv(tmp, out); 3500 mv(out, zr); 3501 blez(len, L_end); 3502 zero_extend(k, k, 32); 3503 slliw(t0, offset, LogBytesPerInt); 3504 add(offset, tmp, t0); 3505 slliw(t0, len, LogBytesPerInt); 3506 add(in, in, t0); 3507 3508 const int unroll = 8; 3509 mv(tmp, unroll); 3510 blt(len, tmp, L_tail_loop); 3511 bind(L_unroll); 3512 for (int i = 0; i < unroll; i++) { 3513 sub(in, in, BytesPerInt); 3514 lwu(t0, Address(in, 0)); 3515 mul(t1, t0, k); 3516 add(t0, t1, out); 3517 sub(offset, offset, BytesPerInt); 3518 lwu(t1, Address(offset, 0)); 3519 add(t0, t0, t1); 3520 sw(t0, Address(offset, 0)); 3521 srli(out, t0, 32); 3522 } 3523 subw(len, len, tmp); 3524 bge(len, tmp, L_unroll); 3525 3526 bind(L_tail_loop); 3527 blez(len, L_end); 3528 sub(in, in, BytesPerInt); 3529 lwu(t0, Address(in, 0)); 3530 mul(t1, t0, k); 3531 add(t0, t1, out); 3532 sub(offset, offset, BytesPerInt); 3533 lwu(t1, Address(offset, 0)); 3534 add(t0, t0, t1); 3535 sw(t0, Address(offset, 0)); 3536 srli(out, t0, 32); 3537 subw(len, len, 1); 3538 j(L_tail_loop); 3539 3540 bind(L_end); 3541 } 3542 3543 // add two unsigned input and output carry 3544 void MacroAssembler::cad(Register dst, Register src1, Register src2, Register carry) 3545 { 3546 assert_different_registers(dst, carry); 3547 assert_different_registers(dst, src2); 3548 add(dst, src1, src2); 3549 sltu(carry, dst, src2); 3550 } 3551 3552 // add two input with carry 3553 void MacroAssembler::adc(Register dst, Register src1, Register src2, Register carry) { 3554 assert_different_registers(dst, carry); 3555 add(dst, src1, src2); 3556 add(dst, dst, carry); 3557 } 3558 3559 // add two unsigned input with carry and output carry 3560 void MacroAssembler::cadc(Register dst, Register src1, Register src2, Register carry) { 3561 assert_different_registers(dst, src2); 3562 adc(dst, src1, src2, carry); 3563 sltu(carry, dst, src2); 3564 } 3565 3566 void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo, 3567 Register src1, Register src2, Register carry) { 3568 cad(dest_lo, dest_lo, src1, carry); 3569 add(dest_hi, dest_hi, carry); 3570 cad(dest_lo, dest_lo, src2, carry); 3571 add(final_dest_hi, dest_hi, carry); 3572 } 3573 3574 /** 3575 * Multiply 32 bit by 32 bit first loop. 3576 */ 3577 void MacroAssembler::multiply_32_x_32_loop(Register x, Register xstart, Register x_xstart, 3578 Register y, Register y_idx, Register z, 3579 Register carry, Register product, 3580 Register idx, Register kdx) { 3581 // jlong carry, x[], y[], z[]; 3582 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) { 3583 // long product = y[idx] * x[xstart] + carry; 3584 // z[kdx] = (int)product; 3585 // carry = product >>> 32; 3586 // } 3587 // z[xstart] = (int)carry; 3588 3589 Label L_first_loop, L_first_loop_exit; 3590 blez(idx, L_first_loop_exit); 3591 3592 shadd(t0, xstart, x, t0, LogBytesPerInt); 3593 lwu(x_xstart, Address(t0, 0)); 3594 3595 bind(L_first_loop); 3596 subw(idx, idx, 1); 3597 shadd(t0, idx, y, t0, LogBytesPerInt); 3598 lwu(y_idx, Address(t0, 0)); 3599 mul(product, x_xstart, y_idx); 3600 add(product, product, carry); 3601 srli(carry, product, 32); 3602 subw(kdx, kdx, 1); 3603 shadd(t0, kdx, z, t0, LogBytesPerInt); 3604 sw(product, Address(t0, 0)); 3605 bgtz(idx, L_first_loop); 3606 3607 bind(L_first_loop_exit); 3608 } 3609 3610 /** 3611 * Multiply 64 bit by 64 bit first loop. 3612 */ 3613 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart, 3614 Register y, Register y_idx, Register z, 3615 Register carry, Register product, 3616 Register idx, Register kdx) { 3617 // 3618 // jlong carry, x[], y[], z[]; 3619 // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) { 3620 // huge_128 product = y[idx] * x[xstart] + carry; 3621 // z[kdx] = (jlong)product; 3622 // carry = (jlong)(product >>> 64); 3623 // } 3624 // z[xstart] = carry; 3625 // 3626 3627 Label L_first_loop, L_first_loop_exit; 3628 Label L_one_x, L_one_y, L_multiply; 3629 3630 subw(xstart, xstart, 1); 3631 bltz(xstart, L_one_x); 3632 3633 shadd(t0, xstart, x, t0, LogBytesPerInt); 3634 ld(x_xstart, Address(t0, 0)); 3635 ror_imm(x_xstart, x_xstart, 32); // convert big-endian to little-endian 3636 3637 bind(L_first_loop); 3638 subw(idx, idx, 1); 3639 bltz(idx, L_first_loop_exit); 3640 subw(idx, idx, 1); 3641 bltz(idx, L_one_y); 3642 3643 shadd(t0, idx, y, t0, LogBytesPerInt); 3644 ld(y_idx, Address(t0, 0)); 3645 ror_imm(y_idx, y_idx, 32); // convert big-endian to little-endian 3646 bind(L_multiply); 3647 3648 mulhu(t0, x_xstart, y_idx); 3649 mul(product, x_xstart, y_idx); 3650 cad(product, product, carry, t1); 3651 adc(carry, t0, zr, t1); 3652 3653 subw(kdx, kdx, 2); 3654 ror_imm(product, product, 32); // back to big-endian 3655 shadd(t0, kdx, z, t0, LogBytesPerInt); 3656 sd(product, Address(t0, 0)); 3657 3658 j(L_first_loop); 3659 3660 bind(L_one_y); 3661 lwu(y_idx, Address(y, 0)); 3662 j(L_multiply); 3663 3664 bind(L_one_x); 3665 lwu(x_xstart, Address(x, 0)); 3666 j(L_first_loop); 3667 3668 bind(L_first_loop_exit); 3669 } 3670 3671 /** 3672 * Multiply 128 bit by 128 bit. Unrolled inner loop. 3673 * 3674 */ 3675 void MacroAssembler::multiply_128_x_128_loop(Register y, Register z, 3676 Register carry, Register carry2, 3677 Register idx, Register jdx, 3678 Register yz_idx1, Register yz_idx2, 3679 Register tmp, Register tmp3, Register tmp4, 3680 Register tmp6, Register product_hi) { 3681 // jlong carry, x[], y[], z[]; 3682 // int kdx = xstart+1; 3683 // for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop 3684 // huge_128 tmp3 = (y[idx+1] * product_hi) + z[kdx+idx+1] + carry; 3685 // jlong carry2 = (jlong)(tmp3 >>> 64); 3686 // huge_128 tmp4 = (y[idx] * product_hi) + z[kdx+idx] + carry2; 3687 // carry = (jlong)(tmp4 >>> 64); 3688 // z[kdx+idx+1] = (jlong)tmp3; 3689 // z[kdx+idx] = (jlong)tmp4; 3690 // } 3691 // idx += 2; 3692 // if (idx > 0) { 3693 // yz_idx1 = (y[idx] * product_hi) + z[kdx+idx] + carry; 3694 // z[kdx+idx] = (jlong)yz_idx1; 3695 // carry = (jlong)(yz_idx1 >>> 64); 3696 // } 3697 // 3698 3699 Label L_third_loop, L_third_loop_exit, L_post_third_loop_done; 3700 3701 srliw(jdx, idx, 2); 3702 3703 bind(L_third_loop); 3704 3705 subw(jdx, jdx, 1); 3706 bltz(jdx, L_third_loop_exit); 3707 subw(idx, idx, 4); 3708 3709 shadd(t0, idx, y, t0, LogBytesPerInt); 3710 ld(yz_idx2, Address(t0, 0)); 3711 ld(yz_idx1, Address(t0, wordSize)); 3712 3713 shadd(tmp6, idx, z, t0, LogBytesPerInt); 3714 3715 ror_imm(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian 3716 ror_imm(yz_idx2, yz_idx2, 32); 3717 3718 ld(t1, Address(tmp6, 0)); 3719 ld(t0, Address(tmp6, wordSize)); 3720 3721 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3 3722 mulhu(tmp4, product_hi, yz_idx1); 3723 3724 ror_imm(t0, t0, 32, tmp); // convert big-endian to little-endian 3725 ror_imm(t1, t1, 32, tmp); 3726 3727 mul(tmp, product_hi, yz_idx2); // yz_idx2 * product_hi -> carry2:tmp 3728 mulhu(carry2, product_hi, yz_idx2); 3729 3730 cad(tmp3, tmp3, carry, carry); 3731 adc(tmp4, tmp4, zr, carry); 3732 cad(tmp3, tmp3, t0, t0); 3733 cadc(tmp4, tmp4, tmp, t0); 3734 adc(carry, carry2, zr, t0); 3735 cad(tmp4, tmp4, t1, carry2); 3736 adc(carry, carry, zr, carry2); 3737 3738 ror_imm(tmp3, tmp3, 32); // convert little-endian to big-endian 3739 ror_imm(tmp4, tmp4, 32); 3740 sd(tmp4, Address(tmp6, 0)); 3741 sd(tmp3, Address(tmp6, wordSize)); 3742 3743 j(L_third_loop); 3744 3745 bind(L_third_loop_exit); 3746 3747 andi(idx, idx, 0x3); 3748 beqz(idx, L_post_third_loop_done); 3749 3750 Label L_check_1; 3751 subw(idx, idx, 2); 3752 bltz(idx, L_check_1); 3753 3754 shadd(t0, idx, y, t0, LogBytesPerInt); 3755 ld(yz_idx1, Address(t0, 0)); 3756 ror_imm(yz_idx1, yz_idx1, 32); 3757 3758 mul(tmp3, product_hi, yz_idx1); // yz_idx1 * product_hi -> tmp4:tmp3 3759 mulhu(tmp4, product_hi, yz_idx1); 3760 3761 shadd(t0, idx, z, t0, LogBytesPerInt); 3762 ld(yz_idx2, Address(t0, 0)); 3763 ror_imm(yz_idx2, yz_idx2, 32, tmp); 3764 3765 add2_with_carry(carry, tmp4, tmp3, carry, yz_idx2, tmp); 3766 3767 ror_imm(tmp3, tmp3, 32, tmp); 3768 sd(tmp3, Address(t0, 0)); 3769 3770 bind(L_check_1); 3771 3772 andi(idx, idx, 0x1); 3773 subw(idx, idx, 1); 3774 bltz(idx, L_post_third_loop_done); 3775 shadd(t0, idx, y, t0, LogBytesPerInt); 3776 lwu(tmp4, Address(t0, 0)); 3777 mul(tmp3, tmp4, product_hi); // tmp4 * product_hi -> carry2:tmp3 3778 mulhu(carry2, tmp4, product_hi); 3779 3780 shadd(t0, idx, z, t0, LogBytesPerInt); 3781 lwu(tmp4, Address(t0, 0)); 3782 3783 add2_with_carry(carry2, carry2, tmp3, tmp4, carry, t0); 3784 3785 shadd(t0, idx, z, t0, LogBytesPerInt); 3786 sw(tmp3, Address(t0, 0)); 3787 3788 slli(t0, carry2, 32); 3789 srli(carry, tmp3, 32); 3790 orr(carry, carry, t0); 3791 3792 bind(L_post_third_loop_done); 3793 } 3794 3795 /** 3796 * Code for BigInteger::multiplyToLen() intrinsic. 3797 * 3798 * x10: x 3799 * x11: xlen 3800 * x12: y 3801 * x13: ylen 3802 * x14: z 3803 * x15: zlen 3804 * x16: tmp1 3805 * x17: tmp2 3806 * x7: tmp3 3807 * x28: tmp4 3808 * x29: tmp5 3809 * x30: tmp6 3810 * x31: tmp7 3811 */ 3812 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen, 3813 Register z, Register zlen, 3814 Register tmp1, Register tmp2, Register tmp3, Register tmp4, 3815 Register tmp5, Register tmp6, Register product_hi) { 3816 assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6); 3817 3818 const Register idx = tmp1; 3819 const Register kdx = tmp2; 3820 const Register xstart = tmp3; 3821 3822 const Register y_idx = tmp4; 3823 const Register carry = tmp5; 3824 const Register product = xlen; 3825 const Register x_xstart = zlen; // reuse register 3826 3827 mv(idx, ylen); // idx = ylen; 3828 mv(kdx, zlen); // kdx = xlen+ylen; 3829 mv(carry, zr); // carry = 0; 3830 3831 Label L_multiply_64_x_64_loop, L_done; 3832 3833 subw(xstart, xlen, 1); 3834 bltz(xstart, L_done); 3835 3836 const Register jdx = tmp1; 3837 3838 if (AvoidUnalignedAccesses) { 3839 // Check if x and y are both 8-byte aligned. 3840 orr(t0, xlen, ylen); 3841 test_bit(t0, t0, 0); 3842 beqz(t0, L_multiply_64_x_64_loop); 3843 3844 multiply_32_x_32_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 3845 shadd(t0, xstart, z, t0, LogBytesPerInt); 3846 sw(carry, Address(t0, 0)); 3847 3848 Label L_second_loop_unaligned; 3849 bind(L_second_loop_unaligned); 3850 mv(carry, zr); 3851 mv(jdx, ylen); 3852 subw(xstart, xstart, 1); 3853 bltz(xstart, L_done); 3854 sub(sp, sp, 2 * wordSize); 3855 sd(z, Address(sp, 0)); 3856 sd(zr, Address(sp, wordSize)); 3857 shadd(t0, xstart, z, t0, LogBytesPerInt); 3858 addi(z, t0, 4); 3859 shadd(t0, xstart, x, t0, LogBytesPerInt); 3860 lwu(product, Address(t0, 0)); 3861 Label L_third_loop, L_third_loop_exit; 3862 3863 blez(jdx, L_third_loop_exit); 3864 3865 bind(L_third_loop); 3866 subw(jdx, jdx, 1); 3867 shadd(t0, jdx, y, t0, LogBytesPerInt); 3868 lwu(t0, Address(t0, 0)); 3869 mul(t1, t0, product); 3870 add(t0, t1, carry); 3871 shadd(tmp6, jdx, z, t1, LogBytesPerInt); 3872 lwu(t1, Address(tmp6, 0)); 3873 add(t0, t0, t1); 3874 sw(t0, Address(tmp6, 0)); 3875 srli(carry, t0, 32); 3876 bgtz(jdx, L_third_loop); 3877 3878 bind(L_third_loop_exit); 3879 ld(z, Address(sp, 0)); 3880 addi(sp, sp, 2 * wordSize); 3881 shadd(t0, xstart, z, t0, LogBytesPerInt); 3882 sw(carry, Address(t0, 0)); 3883 3884 j(L_second_loop_unaligned); 3885 } 3886 3887 bind(L_multiply_64_x_64_loop); 3888 multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx); 3889 3890 Label L_second_loop_aligned; 3891 beqz(kdx, L_second_loop_aligned); 3892 3893 Label L_carry; 3894 subw(kdx, kdx, 1); 3895 beqz(kdx, L_carry); 3896 3897 shadd(t0, kdx, z, t0, LogBytesPerInt); 3898 sw(carry, Address(t0, 0)); 3899 srli(carry, carry, 32); 3900 subw(kdx, kdx, 1); 3901 3902 bind(L_carry); 3903 shadd(t0, kdx, z, t0, LogBytesPerInt); 3904 sw(carry, Address(t0, 0)); 3905 3906 // Second and third (nested) loops. 3907 // 3908 // for (int i = xstart-1; i >= 0; i--) { // Second loop 3909 // carry = 0; 3910 // for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop 3911 // long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) + 3912 // (z[k] & LONG_MASK) + carry; 3913 // z[k] = (int)product; 3914 // carry = product >>> 32; 3915 // } 3916 // z[i] = (int)carry; 3917 // } 3918 // 3919 // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = product_hi 3920 3921 bind(L_second_loop_aligned); 3922 mv(carry, zr); // carry = 0; 3923 mv(jdx, ylen); // j = ystart+1 3924 3925 subw(xstart, xstart, 1); // i = xstart-1; 3926 bltz(xstart, L_done); 3927 3928 sub(sp, sp, 4 * wordSize); 3929 sd(z, Address(sp, 0)); 3930 3931 Label L_last_x; 3932 shadd(t0, xstart, z, t0, LogBytesPerInt); 3933 addi(z, t0, 4); 3934 subw(xstart, xstart, 1); // i = xstart-1; 3935 bltz(xstart, L_last_x); 3936 3937 shadd(t0, xstart, x, t0, LogBytesPerInt); 3938 ld(product_hi, Address(t0, 0)); 3939 ror_imm(product_hi, product_hi, 32); // convert big-endian to little-endian 3940 3941 Label L_third_loop_prologue; 3942 bind(L_third_loop_prologue); 3943 3944 sd(ylen, Address(sp, wordSize)); 3945 sd(x, Address(sp, 2 * wordSize)); 3946 sd(xstart, Address(sp, 3 * wordSize)); 3947 multiply_128_x_128_loop(y, z, carry, x, jdx, ylen, product, 3948 tmp2, x_xstart, tmp3, tmp4, tmp6, product_hi); 3949 ld(z, Address(sp, 0)); 3950 ld(ylen, Address(sp, wordSize)); 3951 ld(x, Address(sp, 2 * wordSize)); 3952 ld(xlen, Address(sp, 3 * wordSize)); // copy old xstart -> xlen 3953 addi(sp, sp, 4 * wordSize); 3954 3955 addiw(tmp3, xlen, 1); 3956 shadd(t0, tmp3, z, t0, LogBytesPerInt); 3957 sw(carry, Address(t0, 0)); 3958 3959 subw(tmp3, tmp3, 1); 3960 bltz(tmp3, L_done); 3961 3962 srli(carry, carry, 32); 3963 shadd(t0, tmp3, z, t0, LogBytesPerInt); 3964 sw(carry, Address(t0, 0)); 3965 j(L_second_loop_aligned); 3966 3967 // Next infrequent code is moved outside loops. 3968 bind(L_last_x); 3969 lwu(product_hi, Address(x, 0)); 3970 j(L_third_loop_prologue); 3971 3972 bind(L_done); 3973 } 3974 #endif 3975 3976 // Count bits of trailing zero chars from lsb to msb until first non-zero element. 3977 // For LL case, one byte for one element, so shift 8 bits once, and for other case, 3978 // shift 16 bits once. 3979 void MacroAssembler::ctzc_bit(Register Rd, Register Rs, bool isLL, Register tmp1, Register tmp2) { 3980 if (UseZbb) { 3981 assert_different_registers(Rd, Rs, tmp1); 3982 int step = isLL ? 8 : 16; 3983 ctz(Rd, Rs); 3984 andi(tmp1, Rd, step - 1); 3985 sub(Rd, Rd, tmp1); 3986 return; 3987 } 3988 3989 assert_different_registers(Rd, Rs, tmp1, tmp2); 3990 Label Loop; 3991 int step = isLL ? 8 : 16; 3992 mv(Rd, -step); 3993 mv(tmp2, Rs); 3994 3995 bind(Loop); 3996 addi(Rd, Rd, step); 3997 andi(tmp1, tmp2, ((1 << step) - 1)); 3998 srli(tmp2, tmp2, step); 3999 beqz(tmp1, Loop); 4000 } 4001 4002 // This instruction reads adjacent 4 bytes from the lower half of source register, 4003 // inflate into a register, for example: 4004 // Rs: A7A6A5A4A3A2A1A0 4005 // Rd: 00A300A200A100A0 4006 void MacroAssembler::inflate_lo32(Register Rd, Register Rs, Register tmp1, Register tmp2) { 4007 assert_different_registers(Rd, Rs, tmp1, tmp2); 4008 4009 mv(tmp1, 0xFF000000); // first byte mask at lower word 4010 andr(Rd, Rs, tmp1); 4011 for (int i = 0; i < 2; i++) { 4012 slli(Rd, Rd, wordSize); 4013 srli(tmp1, tmp1, wordSize); 4014 andr(tmp2, Rs, tmp1); 4015 orr(Rd, Rd, tmp2); 4016 } 4017 slli(Rd, Rd, wordSize); 4018 andi(tmp2, Rs, 0xFF); // last byte mask at lower word 4019 orr(Rd, Rd, tmp2); 4020 } 4021 4022 // This instruction reads adjacent 4 bytes from the upper half of source register, 4023 // inflate into a register, for example: 4024 // Rs: A7A6A5A4A3A2A1A0 4025 // Rd: 00A700A600A500A4 4026 void MacroAssembler::inflate_hi32(Register Rd, Register Rs, Register tmp1, Register tmp2) { 4027 assert_different_registers(Rd, Rs, tmp1, tmp2); 4028 srli(Rs, Rs, 32); // only upper 32 bits are needed 4029 inflate_lo32(Rd, Rs, tmp1, tmp2); 4030 } 4031 4032 // The size of the blocks erased by the zero_blocks stub. We must 4033 // handle anything smaller than this ourselves in zero_words(). 4034 const int MacroAssembler::zero_words_block_size = 8; 4035 4036 // zero_words() is used by C2 ClearArray patterns. It is as small as 4037 // possible, handling small word counts locally and delegating 4038 // anything larger to the zero_blocks stub. It is expanded many times 4039 // in compiled code, so it is important to keep it short. 4040 4041 // ptr: Address of a buffer to be zeroed. 4042 // cnt: Count in HeapWords. 4043 // 4044 // ptr, cnt, and t0 are clobbered. 4045 address MacroAssembler::zero_words(Register ptr, Register cnt) { 4046 assert(is_power_of_2(zero_words_block_size), "adjust this"); 4047 assert(ptr == x28 && cnt == x29, "mismatch in register usage"); 4048 assert_different_registers(cnt, t0); 4049 4050 BLOCK_COMMENT("zero_words {"); 4051 4052 mv(t0, zero_words_block_size); 4053 Label around, done, done16; 4054 bltu(cnt, t0, around); 4055 { 4056 RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::riscv::zero_blocks()); 4057 assert(zero_blocks.target() != nullptr, "zero_blocks stub has not been generated"); 4058 if (StubRoutines::riscv::complete()) { 4059 address tpc = trampoline_call(zero_blocks); 4060 if (tpc == nullptr) { 4061 DEBUG_ONLY(reset_labels(around)); 4062 postcond(pc() == badAddress); 4063 return nullptr; 4064 } 4065 } else { 4066 jal(zero_blocks); 4067 } 4068 } 4069 bind(around); 4070 for (int i = zero_words_block_size >> 1; i > 1; i >>= 1) { 4071 Label l; 4072 test_bit(t0, cnt, exact_log2(i)); 4073 beqz(t0, l); 4074 for (int j = 0; j < i; j++) { 4075 sd(zr, Address(ptr, j * wordSize)); 4076 } 4077 addi(ptr, ptr, i * wordSize); 4078 bind(l); 4079 } 4080 { 4081 Label l; 4082 test_bit(t0, cnt, 0); 4083 beqz(t0, l); 4084 sd(zr, Address(ptr, 0)); 4085 bind(l); 4086 } 4087 4088 BLOCK_COMMENT("} zero_words"); 4089 postcond(pc() != badAddress); 4090 return pc(); 4091 } 4092 4093 #define SmallArraySize (18 * BytesPerLong) 4094 4095 // base: Address of a buffer to be zeroed, 8 bytes aligned. 4096 // cnt: Immediate count in HeapWords. 4097 void MacroAssembler::zero_words(Register base, uint64_t cnt) { 4098 assert_different_registers(base, t0, t1); 4099 4100 BLOCK_COMMENT("zero_words {"); 4101 4102 if (cnt <= SmallArraySize / BytesPerLong) { 4103 for (int i = 0; i < (int)cnt; i++) { 4104 sd(zr, Address(base, i * wordSize)); 4105 } 4106 } else { 4107 const int unroll = 8; // Number of sd(zr, adr), instructions we'll unroll 4108 int remainder = cnt % unroll; 4109 for (int i = 0; i < remainder; i++) { 4110 sd(zr, Address(base, i * wordSize)); 4111 } 4112 4113 Label loop; 4114 Register cnt_reg = t0; 4115 Register loop_base = t1; 4116 cnt = cnt - remainder; 4117 mv(cnt_reg, cnt); 4118 add(loop_base, base, remainder * wordSize); 4119 bind(loop); 4120 sub(cnt_reg, cnt_reg, unroll); 4121 for (int i = 0; i < unroll; i++) { 4122 sd(zr, Address(loop_base, i * wordSize)); 4123 } 4124 add(loop_base, loop_base, unroll * wordSize); 4125 bnez(cnt_reg, loop); 4126 } 4127 4128 BLOCK_COMMENT("} zero_words"); 4129 } 4130 4131 // base: Address of a buffer to be filled, 8 bytes aligned. 4132 // cnt: Count in 8-byte unit. 4133 // value: Value to be filled with. 4134 // base will point to the end of the buffer after filling. 4135 void MacroAssembler::fill_words(Register base, Register cnt, Register value) { 4136 // Algorithm: 4137 // 4138 // t0 = cnt & 7 4139 // cnt -= t0 4140 // p += t0 4141 // switch (t0): 4142 // switch start: 4143 // do while cnt 4144 // cnt -= 8 4145 // p[-8] = value 4146 // case 7: 4147 // p[-7] = value 4148 // case 6: 4149 // p[-6] = value 4150 // // ... 4151 // case 1: 4152 // p[-1] = value 4153 // case 0: 4154 // p += 8 4155 // do-while end 4156 // switch end 4157 4158 assert_different_registers(base, cnt, value, t0, t1); 4159 4160 Label fini, skip, entry, loop; 4161 const int unroll = 8; // Number of sd instructions we'll unroll 4162 4163 beqz(cnt, fini); 4164 4165 andi(t0, cnt, unroll - 1); 4166 sub(cnt, cnt, t0); 4167 // align 8, so first sd n % 8 = mod, next loop sd 8 * n. 4168 shadd(base, t0, base, t1, 3); 4169 la(t1, entry); 4170 slli(t0, t0, 2); // sd_inst_nums * 4; t0 is cnt % 8, so t1 = t1 - sd_inst_nums * 4, 4 is sizeof(inst) 4171 sub(t1, t1, t0); 4172 jr(t1); 4173 4174 bind(loop); 4175 add(base, base, unroll * 8); 4176 for (int i = -unroll; i < 0; i++) { 4177 sd(value, Address(base, i * 8)); 4178 } 4179 bind(entry); 4180 sub(cnt, cnt, unroll); 4181 bgez(cnt, loop); 4182 4183 bind(fini); 4184 } 4185 4186 // Zero blocks of memory by using CBO.ZERO. 4187 // 4188 // Aligns the base address first sufficiently for CBO.ZERO, then uses 4189 // CBO.ZERO repeatedly for every full block. cnt is the size to be 4190 // zeroed in HeapWords. Returns the count of words left to be zeroed 4191 // in cnt. 4192 // 4193 // NOTE: This is intended to be used in the zero_blocks() stub. If 4194 // you want to use it elsewhere, note that cnt must be >= CacheLineSize. 4195 void MacroAssembler::zero_dcache_blocks(Register base, Register cnt, Register tmp1, Register tmp2) { 4196 Label initial_table_end, loop; 4197 4198 // Align base with cache line size. 4199 neg(tmp1, base); 4200 andi(tmp1, tmp1, CacheLineSize - 1); 4201 4202 // tmp1: the number of bytes to be filled to align the base with cache line size. 4203 add(base, base, tmp1); 4204 srai(tmp2, tmp1, 3); 4205 sub(cnt, cnt, tmp2); 4206 srli(tmp2, tmp1, 1); 4207 la(tmp1, initial_table_end); 4208 sub(tmp2, tmp1, tmp2); 4209 jr(tmp2); 4210 for (int i = -CacheLineSize + wordSize; i < 0; i += wordSize) { 4211 sd(zr, Address(base, i)); 4212 } 4213 bind(initial_table_end); 4214 4215 mv(tmp1, CacheLineSize / wordSize); 4216 bind(loop); 4217 cbo_zero(base); 4218 sub(cnt, cnt, tmp1); 4219 add(base, base, CacheLineSize); 4220 bge(cnt, tmp1, loop); 4221 } 4222 4223 // java.lang.Math.round(float a) 4224 // Returns the closest int to the argument, with ties rounding to positive infinity. 4225 void MacroAssembler::java_round_float(Register dst, FloatRegister src, FloatRegister ftmp) { 4226 // this instructions calling sequence provides performance improvement on all tested devices; 4227 // don't change it without re-verification 4228 Label done; 4229 mv(t0, jint_cast(0.5f)); 4230 fmv_w_x(ftmp, t0); 4231 4232 // dst = 0 if NaN 4233 feq_s(t0, src, src); // replacing fclass with feq as performance optimization 4234 mv(dst, zr); 4235 beqz(t0, done); 4236 4237 // dst = (src + 0.5f) rounded down towards negative infinity 4238 // Adding 0.5f to some floats exceeds the precision limits for a float and rounding takes place. 4239 // RDN is required for fadd_s, RNE gives incorrect results: 4240 // -------------------------------------------------------------------- 4241 // fadd.s rne (src + 0.5f): src = 8388609.000000 ftmp = 8388610.000000 4242 // fcvt.w.s rdn: ftmp = 8388610.000000 dst = 8388610 4243 // -------------------------------------------------------------------- 4244 // fadd.s rdn (src + 0.5f): src = 8388609.000000 ftmp = 8388609.000000 4245 // fcvt.w.s rdn: ftmp = 8388609.000000 dst = 8388609 4246 // -------------------------------------------------------------------- 4247 fadd_s(ftmp, src, ftmp, RoundingMode::rdn); 4248 fcvt_w_s(dst, ftmp, RoundingMode::rdn); 4249 4250 bind(done); 4251 } 4252 4253 // java.lang.Math.round(double a) 4254 // Returns the closest long to the argument, with ties rounding to positive infinity. 4255 void MacroAssembler::java_round_double(Register dst, FloatRegister src, FloatRegister ftmp) { 4256 // this instructions calling sequence provides performance improvement on all tested devices; 4257 // don't change it without re-verification 4258 Label done; 4259 mv(t0, julong_cast(0.5)); 4260 fmv_d_x(ftmp, t0); 4261 4262 // dst = 0 if NaN 4263 feq_d(t0, src, src); // replacing fclass with feq as performance optimization 4264 mv(dst, zr); 4265 beqz(t0, done); 4266 4267 // dst = (src + 0.5) rounded down towards negative infinity 4268 fadd_d(ftmp, src, ftmp, RoundingMode::rdn); // RDN is required here otherwise some inputs produce incorrect results 4269 fcvt_l_d(dst, ftmp, RoundingMode::rdn); 4270 4271 bind(done); 4272 } 4273 4274 #define FCVT_SAFE(FLOATCVT, FLOATSIG) \ 4275 void MacroAssembler::FLOATCVT##_safe(Register dst, FloatRegister src, Register tmp) { \ 4276 Label done; \ 4277 assert_different_registers(dst, tmp); \ 4278 fclass_##FLOATSIG(tmp, src); \ 4279 mv(dst, zr); \ 4280 /* check if src is NaN */ \ 4281 andi(tmp, tmp, 0b1100000000); \ 4282 bnez(tmp, done); \ 4283 FLOATCVT(dst, src); \ 4284 bind(done); \ 4285 } 4286 4287 FCVT_SAFE(fcvt_w_s, s); 4288 FCVT_SAFE(fcvt_l_s, s); 4289 FCVT_SAFE(fcvt_w_d, d); 4290 FCVT_SAFE(fcvt_l_d, d); 4291 4292 #undef FCVT_SAFE 4293 4294 #define FCMP(FLOATTYPE, FLOATSIG) \ 4295 void MacroAssembler::FLOATTYPE##_compare(Register result, FloatRegister Rs1, \ 4296 FloatRegister Rs2, int unordered_result) { \ 4297 Label Ldone; \ 4298 if (unordered_result < 0) { \ 4299 /* we want -1 for unordered or less than, 0 for equal and 1 for greater than. */ \ 4300 /* installs 1 if gt else 0 */ \ 4301 flt_##FLOATSIG(result, Rs2, Rs1); \ 4302 /* Rs1 > Rs2, install 1 */ \ 4303 bgtz(result, Ldone); \ 4304 feq_##FLOATSIG(result, Rs1, Rs2); \ 4305 addi(result, result, -1); \ 4306 /* Rs1 = Rs2, install 0 */ \ 4307 /* NaN or Rs1 < Rs2, install -1 */ \ 4308 bind(Ldone); \ 4309 } else { \ 4310 /* we want -1 for less than, 0 for equal and 1 for unordered or greater than. */ \ 4311 /* installs 1 if gt or unordered else 0 */ \ 4312 flt_##FLOATSIG(result, Rs1, Rs2); \ 4313 /* Rs1 < Rs2, install -1 */ \ 4314 bgtz(result, Ldone); \ 4315 feq_##FLOATSIG(result, Rs1, Rs2); \ 4316 addi(result, result, -1); \ 4317 /* Rs1 = Rs2, install 0 */ \ 4318 /* NaN or Rs1 > Rs2, install 1 */ \ 4319 bind(Ldone); \ 4320 neg(result, result); \ 4321 } \ 4322 } 4323 4324 FCMP(float, s); 4325 FCMP(double, d); 4326 4327 #undef FCMP 4328 4329 // Zero words; len is in bytes 4330 // Destroys all registers except addr 4331 // len must be a nonzero multiple of wordSize 4332 void MacroAssembler::zero_memory(Register addr, Register len, Register tmp) { 4333 assert_different_registers(addr, len, tmp, t0, t1); 4334 4335 #ifdef ASSERT 4336 { 4337 Label L; 4338 andi(t0, len, BytesPerWord - 1); 4339 beqz(t0, L); 4340 stop("len is not a multiple of BytesPerWord"); 4341 bind(L); 4342 } 4343 #endif // ASSERT 4344 4345 #ifndef PRODUCT 4346 block_comment("zero memory"); 4347 #endif // PRODUCT 4348 4349 Label loop; 4350 Label entry; 4351 4352 // Algorithm: 4353 // 4354 // t0 = cnt & 7 4355 // cnt -= t0 4356 // p += t0 4357 // switch (t0) { 4358 // do { 4359 // cnt -= 8 4360 // p[-8] = 0 4361 // case 7: 4362 // p[-7] = 0 4363 // case 6: 4364 // p[-6] = 0 4365 // ... 4366 // case 1: 4367 // p[-1] = 0 4368 // case 0: 4369 // p += 8 4370 // } while (cnt) 4371 // } 4372 4373 const int unroll = 8; // Number of sd(zr) instructions we'll unroll 4374 4375 srli(len, len, LogBytesPerWord); 4376 andi(t0, len, unroll - 1); // t0 = cnt % unroll 4377 sub(len, len, t0); // cnt -= unroll 4378 // tmp always points to the end of the region we're about to zero 4379 shadd(tmp, t0, addr, t1, LogBytesPerWord); 4380 la(t1, entry); 4381 slli(t0, t0, 2); 4382 sub(t1, t1, t0); 4383 jr(t1); 4384 bind(loop); 4385 sub(len, len, unroll); 4386 for (int i = -unroll; i < 0; i++) { 4387 sd(zr, Address(tmp, i * wordSize)); 4388 } 4389 bind(entry); 4390 add(tmp, tmp, unroll * wordSize); 4391 bnez(len, loop); 4392 } 4393 4394 // shift left by shamt and add 4395 // Rd = (Rs1 << shamt) + Rs2 4396 void MacroAssembler::shadd(Register Rd, Register Rs1, Register Rs2, Register tmp, int shamt) { 4397 if (UseZba) { 4398 if (shamt == 1) { 4399 sh1add(Rd, Rs1, Rs2); 4400 return; 4401 } else if (shamt == 2) { 4402 sh2add(Rd, Rs1, Rs2); 4403 return; 4404 } else if (shamt == 3) { 4405 sh3add(Rd, Rs1, Rs2); 4406 return; 4407 } 4408 } 4409 4410 if (shamt != 0) { 4411 slli(tmp, Rs1, shamt); 4412 add(Rd, Rs2, tmp); 4413 } else { 4414 add(Rd, Rs1, Rs2); 4415 } 4416 } 4417 4418 void MacroAssembler::zero_extend(Register dst, Register src, int bits) { 4419 if (UseZba && bits == 32) { 4420 zext_w(dst, src); 4421 return; 4422 } 4423 4424 if (UseZbb && bits == 16) { 4425 zext_h(dst, src); 4426 return; 4427 } 4428 4429 if (bits == 8) { 4430 zext_b(dst, src); 4431 } else { 4432 slli(dst, src, XLEN - bits); 4433 srli(dst, dst, XLEN - bits); 4434 } 4435 } 4436 4437 void MacroAssembler::sign_extend(Register dst, Register src, int bits) { 4438 if (UseZbb) { 4439 if (bits == 8) { 4440 sext_b(dst, src); 4441 return; 4442 } else if (bits == 16) { 4443 sext_h(dst, src); 4444 return; 4445 } 4446 } 4447 4448 if (bits == 32) { 4449 sext_w(dst, src); 4450 } else { 4451 slli(dst, src, XLEN - bits); 4452 srai(dst, dst, XLEN - bits); 4453 } 4454 } 4455 4456 void MacroAssembler::cmp_l2i(Register dst, Register src1, Register src2, Register tmp) 4457 { 4458 if (src1 == src2) { 4459 mv(dst, zr); 4460 return; 4461 } 4462 Label done; 4463 Register left = src1; 4464 Register right = src2; 4465 if (dst == src1) { 4466 assert_different_registers(dst, src2, tmp); 4467 mv(tmp, src1); 4468 left = tmp; 4469 } else if (dst == src2) { 4470 assert_different_registers(dst, src1, tmp); 4471 mv(tmp, src2); 4472 right = tmp; 4473 } 4474 4475 // installs 1 if gt else 0 4476 slt(dst, right, left); 4477 bnez(dst, done); 4478 slt(dst, left, right); 4479 // dst = -1 if lt; else if eq , dst = 0 4480 neg(dst, dst); 4481 bind(done); 4482 } 4483 4484 // The java_calling_convention describes stack locations as ideal slots on 4485 // a frame with no abi restrictions. Since we must observe abi restrictions 4486 // (like the placement of the register window) the slots must be biased by 4487 // the following value. 4488 static int reg2offset_in(VMReg r) { 4489 // Account for saved fp and ra 4490 // This should really be in_preserve_stack_slots 4491 return r->reg2stack() * VMRegImpl::stack_slot_size; 4492 } 4493 4494 static int reg2offset_out(VMReg r) { 4495 return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size; 4496 } 4497 4498 // On 64 bit we will store integer like items to the stack as 4499 // 64 bits items (riscv64 abi) even though java would only store 4500 // 32bits for a parameter. On 32bit it will simply be 32 bits 4501 // So this routine will do 32->32 on 32bit and 32->64 on 64bit 4502 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp) { 4503 if (src.first()->is_stack()) { 4504 if (dst.first()->is_stack()) { 4505 // stack to stack 4506 ld(tmp, Address(fp, reg2offset_in(src.first()))); 4507 sd(tmp, Address(sp, reg2offset_out(dst.first()))); 4508 } else { 4509 // stack to reg 4510 lw(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); 4511 } 4512 } else if (dst.first()->is_stack()) { 4513 // reg to stack 4514 sd(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first()))); 4515 } else { 4516 if (dst.first() != src.first()) { 4517 sign_extend(dst.first()->as_Register(), src.first()->as_Register(), 32); 4518 } 4519 } 4520 } 4521 4522 // An oop arg. Must pass a handle not the oop itself 4523 void MacroAssembler::object_move(OopMap* map, 4524 int oop_handle_offset, 4525 int framesize_in_slots, 4526 VMRegPair src, 4527 VMRegPair dst, 4528 bool is_receiver, 4529 int* receiver_offset) { 4530 assert_cond(map != nullptr && receiver_offset != nullptr); 4531 4532 // must pass a handle. First figure out the location we use as a handle 4533 Register rHandle = dst.first()->is_stack() ? t1 : dst.first()->as_Register(); 4534 4535 // See if oop is null if it is we need no handle 4536 4537 if (src.first()->is_stack()) { 4538 // Oop is already on the stack as an argument 4539 int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots(); 4540 map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots)); 4541 if (is_receiver) { 4542 *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size; 4543 } 4544 4545 ld(t0, Address(fp, reg2offset_in(src.first()))); 4546 la(rHandle, Address(fp, reg2offset_in(src.first()))); 4547 // conditionally move a null 4548 Label notZero1; 4549 bnez(t0, notZero1); 4550 mv(rHandle, zr); 4551 bind(notZero1); 4552 } else { 4553 4554 // Oop is in a register we must store it to the space we reserve 4555 // on the stack for oop_handles and pass a handle if oop is non-null 4556 4557 const Register rOop = src.first()->as_Register(); 4558 int oop_slot = -1; 4559 if (rOop == j_rarg0) { 4560 oop_slot = 0; 4561 } else if (rOop == j_rarg1) { 4562 oop_slot = 1; 4563 } else if (rOop == j_rarg2) { 4564 oop_slot = 2; 4565 } else if (rOop == j_rarg3) { 4566 oop_slot = 3; 4567 } else if (rOop == j_rarg4) { 4568 oop_slot = 4; 4569 } else if (rOop == j_rarg5) { 4570 oop_slot = 5; 4571 } else if (rOop == j_rarg6) { 4572 oop_slot = 6; 4573 } else { 4574 assert(rOop == j_rarg7, "wrong register"); 4575 oop_slot = 7; 4576 } 4577 4578 oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset; 4579 int offset = oop_slot * VMRegImpl::stack_slot_size; 4580 4581 map->set_oop(VMRegImpl::stack2reg(oop_slot)); 4582 // Store oop in handle area, may be null 4583 sd(rOop, Address(sp, offset)); 4584 if (is_receiver) { 4585 *receiver_offset = offset; 4586 } 4587 4588 //rOop maybe the same as rHandle 4589 if (rOop == rHandle) { 4590 Label isZero; 4591 beqz(rOop, isZero); 4592 la(rHandle, Address(sp, offset)); 4593 bind(isZero); 4594 } else { 4595 Label notZero2; 4596 la(rHandle, Address(sp, offset)); 4597 bnez(rOop, notZero2); 4598 mv(rHandle, zr); 4599 bind(notZero2); 4600 } 4601 } 4602 4603 // If arg is on the stack then place it otherwise it is already in correct reg. 4604 if (dst.first()->is_stack()) { 4605 sd(rHandle, Address(sp, reg2offset_out(dst.first()))); 4606 } 4607 } 4608 4609 // A float arg may have to do float reg int reg conversion 4610 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp) { 4611 assert(src.first()->is_stack() && dst.first()->is_stack() || 4612 src.first()->is_reg() && dst.first()->is_reg() || 4613 src.first()->is_stack() && dst.first()->is_reg(), "Unexpected error"); 4614 if (src.first()->is_stack()) { 4615 if (dst.first()->is_stack()) { 4616 lwu(tmp, Address(fp, reg2offset_in(src.first()))); 4617 sw(tmp, Address(sp, reg2offset_out(dst.first()))); 4618 } else if (dst.first()->is_Register()) { 4619 lwu(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); 4620 } else { 4621 ShouldNotReachHere(); 4622 } 4623 } else if (src.first() != dst.first()) { 4624 if (src.is_single_phys_reg() && dst.is_single_phys_reg()) { 4625 fmv_s(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister()); 4626 } else { 4627 ShouldNotReachHere(); 4628 } 4629 } 4630 } 4631 4632 // A long move 4633 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp) { 4634 if (src.first()->is_stack()) { 4635 if (dst.first()->is_stack()) { 4636 // stack to stack 4637 ld(tmp, Address(fp, reg2offset_in(src.first()))); 4638 sd(tmp, Address(sp, reg2offset_out(dst.first()))); 4639 } else { 4640 // stack to reg 4641 ld(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); 4642 } 4643 } else if (dst.first()->is_stack()) { 4644 // reg to stack 4645 sd(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first()))); 4646 } else { 4647 if (dst.first() != src.first()) { 4648 mv(dst.first()->as_Register(), src.first()->as_Register()); 4649 } 4650 } 4651 } 4652 4653 // A double move 4654 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) { 4655 assert(src.first()->is_stack() && dst.first()->is_stack() || 4656 src.first()->is_reg() && dst.first()->is_reg() || 4657 src.first()->is_stack() && dst.first()->is_reg(), "Unexpected error"); 4658 if (src.first()->is_stack()) { 4659 if (dst.first()->is_stack()) { 4660 ld(tmp, Address(fp, reg2offset_in(src.first()))); 4661 sd(tmp, Address(sp, reg2offset_out(dst.first()))); 4662 } else if (dst.first()-> is_Register()) { 4663 ld(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first()))); 4664 } else { 4665 ShouldNotReachHere(); 4666 } 4667 } else if (src.first() != dst.first()) { 4668 if (src.is_single_phys_reg() && dst.is_single_phys_reg()) { 4669 fmv_d(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister()); 4670 } else { 4671 ShouldNotReachHere(); 4672 } 4673 } 4674 } 4675 4676 void MacroAssembler::rt_call(address dest, Register tmp) { 4677 CodeBlob *cb = CodeCache::find_blob(dest); 4678 RuntimeAddress target(dest); 4679 if (cb) { 4680 far_call(target); 4681 } else { 4682 relocate(target.rspec(), [&] { 4683 int32_t offset; 4684 la_patchable(tmp, target, offset); 4685 jalr(x1, tmp, offset); 4686 }); 4687 } 4688 } 4689 4690 void MacroAssembler::test_bit(Register Rd, Register Rs, uint32_t bit_pos) { 4691 assert(bit_pos < 64, "invalid bit range"); 4692 if (UseZbs) { 4693 bexti(Rd, Rs, bit_pos); 4694 return; 4695 } 4696 int64_t imm = (int64_t)(1UL << bit_pos); 4697 if (is_simm12(imm)) { 4698 and_imm12(Rd, Rs, imm); 4699 } else { 4700 srli(Rd, Rs, bit_pos); 4701 and_imm12(Rd, Rd, 1); 4702 } 4703 } 4704 4705 // Implements lightweight-locking. 4706 // Branches to slow upon failure to lock the object. 4707 // Falls through upon success. 4708 // 4709 // - obj: the object to be locked 4710 // - hdr: the header, already loaded from obj, will be destroyed 4711 // - tmp1, tmp2: temporary registers, will be destroyed 4712 void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow) { 4713 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking"); 4714 assert_different_registers(obj, hdr, tmp1, tmp2, t0); 4715 4716 // Check if we would have space on lock-stack for the object. 4717 lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset())); 4718 mv(tmp2, (unsigned)LockStack::end_offset()); 4719 bge(tmp1, tmp2, slow, /* is_far */ true); 4720 4721 // Load (object->mark() | 1) into hdr 4722 ori(hdr, hdr, markWord::unlocked_value); 4723 // Clear lock-bits, into tmp2 4724 xori(tmp2, hdr, markWord::unlocked_value); 4725 4726 // Try to swing header from unlocked to locked 4727 Label success; 4728 cmpxchgptr(hdr, tmp2, obj, tmp1, success, &slow); 4729 bind(success); 4730 4731 // After successful lock, push object on lock-stack 4732 lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset())); 4733 add(tmp2, xthread, tmp1); 4734 sd(obj, Address(tmp2, 0)); 4735 addw(tmp1, tmp1, oopSize); 4736 sw(tmp1, Address(xthread, JavaThread::lock_stack_top_offset())); 4737 } 4738 4739 // Implements ligthweight-unlocking. 4740 // Branches to slow upon failure. 4741 // Falls through upon success. 4742 // 4743 // - obj: the object to be unlocked 4744 // - hdr: the (pre-loaded) header of the object 4745 // - tmp1, tmp2: temporary registers 4746 void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow) { 4747 assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking"); 4748 assert_different_registers(obj, hdr, tmp1, tmp2, t0); 4749 4750 #ifdef ASSERT 4751 { 4752 // The following checks rely on the fact that LockStack is only ever modified by 4753 // its owning thread, even if the lock got inflated concurrently; removal of LockStack 4754 // entries after inflation will happen delayed in that case. 4755 4756 // Check for lock-stack underflow. 4757 Label stack_ok; 4758 lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset())); 4759 mv(tmp2, (unsigned)LockStack::start_offset()); 4760 bgt(tmp1, tmp2, stack_ok); 4761 STOP("Lock-stack underflow"); 4762 bind(stack_ok); 4763 } 4764 { 4765 // Check if the top of the lock-stack matches the unlocked object. 4766 Label tos_ok; 4767 subw(tmp1, tmp1, oopSize); 4768 add(tmp1, xthread, tmp1); 4769 ld(tmp1, Address(tmp1, 0)); 4770 beq(tmp1, obj, tos_ok); 4771 STOP("Top of lock-stack does not match the unlocked object"); 4772 bind(tos_ok); 4773 } 4774 { 4775 // Check that hdr is fast-locked. 4776 Label hdr_ok; 4777 andi(tmp1, hdr, markWord::lock_mask_in_place); 4778 beqz(tmp1, hdr_ok); 4779 STOP("Header is not fast-locked"); 4780 bind(hdr_ok); 4781 } 4782 #endif 4783 4784 // Load the new header (unlocked) into tmp1 4785 ori(tmp1, hdr, markWord::unlocked_value); 4786 4787 // Try to swing header from locked to unlocked 4788 Label success; 4789 cmpxchgptr(hdr, tmp1, obj, tmp2, success, &slow); 4790 bind(success); 4791 4792 // After successful unlock, pop object from lock-stack 4793 lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset())); 4794 subw(tmp1, tmp1, oopSize); 4795 #ifdef ASSERT 4796 add(tmp2, xthread, tmp1); 4797 sd(zr, Address(tmp2, 0)); 4798 #endif 4799 sw(tmp1, Address(xthread, JavaThread::lock_stack_top_offset())); 4800 }