1 /*
   2  * Copyright (c) 1997, 2023, Oracle and/or its affiliates. All rights reserved.
   3  * Copyright (c) 2014, 2020, Red Hat Inc. All rights reserved.
   4  * Copyright (c) 2020, 2023, Huawei Technologies Co., Ltd. All rights reserved.
   5  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   6  *
   7  * This code is free software; you can redistribute it and/or modify it
   8  * under the terms of the GNU General Public License version 2 only, as
   9  * published by the Free Software Foundation.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  *
  25  */
  26 
  27 #include "precompiled.hpp"
  28 #include "asm/assembler.hpp"
  29 #include "asm/assembler.inline.hpp"
  30 #include "compiler/disassembler.hpp"
  31 #include "gc/shared/barrierSet.hpp"
  32 #include "gc/shared/barrierSetAssembler.hpp"
  33 #include "gc/shared/cardTable.hpp"
  34 #include "gc/shared/cardTableBarrierSet.hpp"
  35 #include "gc/shared/collectedHeap.hpp"
  36 #include "interpreter/bytecodeHistogram.hpp"
  37 #include "interpreter/interpreter.hpp"
  38 #include "memory/resourceArea.hpp"
  39 #include "memory/universe.hpp"
  40 #include "nativeInst_riscv.hpp"
  41 #include "oops/accessDecorators.hpp"
  42 #include "oops/compressedOops.inline.hpp"
  43 #include "oops/klass.inline.hpp"
  44 #include "oops/oop.hpp"
  45 #include "runtime/interfaceSupport.inline.hpp"
  46 #include "runtime/javaThread.hpp"
  47 #include "runtime/jniHandles.inline.hpp"
  48 #include "runtime/sharedRuntime.hpp"
  49 #include "runtime/stubRoutines.hpp"
  50 #include "utilities/powerOfTwo.hpp"
  51 #ifdef COMPILER2
  52 #include "opto/compile.hpp"
  53 #include "opto/node.hpp"
  54 #include "opto/output.hpp"
  55 #endif
  56 
  57 #ifdef PRODUCT
  58 #define BLOCK_COMMENT(str) /* nothing */
  59 #else
  60 #define BLOCK_COMMENT(str) block_comment(str)
  61 #endif
  62 #define STOP(str) stop(str);
  63 #define BIND(label) bind(label); __ BLOCK_COMMENT(#label ":")
  64 
  65 static void pass_arg0(MacroAssembler* masm, Register arg) {
  66   if (c_rarg0 != arg) {
  67     masm->mv(c_rarg0, arg);
  68   }
  69 }
  70 
  71 static void pass_arg1(MacroAssembler* masm, Register arg) {
  72   if (c_rarg1 != arg) {
  73     masm->mv(c_rarg1, arg);
  74   }
  75 }
  76 
  77 static void pass_arg2(MacroAssembler* masm, Register arg) {
  78   if (c_rarg2 != arg) {
  79     masm->mv(c_rarg2, arg);
  80   }
  81 }
  82 
  83 static void pass_arg3(MacroAssembler* masm, Register arg) {
  84   if (c_rarg3 != arg) {
  85     masm->mv(c_rarg3, arg);
  86   }
  87 }
  88 
  89 void MacroAssembler::push_cont_fastpath(Register java_thread) {
  90   if (!Continuations::enabled()) return;
  91   Label done;
  92   ld(t0, Address(java_thread, JavaThread::cont_fastpath_offset()));
  93   bleu(sp, t0, done);
  94   sd(sp, Address(java_thread, JavaThread::cont_fastpath_offset()));
  95   bind(done);
  96 }
  97 
  98 void MacroAssembler::pop_cont_fastpath(Register java_thread) {
  99   if (!Continuations::enabled()) return;
 100   Label done;
 101   ld(t0, Address(java_thread, JavaThread::cont_fastpath_offset()));
 102   bltu(sp, t0, done);
 103   sd(zr, Address(java_thread, JavaThread::cont_fastpath_offset()));
 104   bind(done);
 105 }
 106 
 107 int MacroAssembler::align(int modulus, int extra_offset) {
 108   CompressibleRegion cr(this);
 109   intptr_t before = offset();
 110   while ((offset() + extra_offset) % modulus != 0) { nop(); }
 111   return (int)(offset() - before);
 112 }
 113 
 114 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 115   call_VM_base(oop_result, noreg, noreg, entry_point, number_of_arguments, check_exceptions);
 116 }
 117 
 118 // Implementation of call_VM versions
 119 
 120 void MacroAssembler::call_VM(Register oop_result,
 121                              address entry_point,
 122                              bool check_exceptions) {
 123   call_VM_helper(oop_result, entry_point, 0, check_exceptions);
 124 }
 125 
 126 void MacroAssembler::call_VM(Register oop_result,
 127                              address entry_point,
 128                              Register arg_1,
 129                              bool check_exceptions) {
 130   pass_arg1(this, arg_1);
 131   call_VM_helper(oop_result, entry_point, 1, check_exceptions);
 132 }
 133 
 134 void MacroAssembler::call_VM(Register oop_result,
 135                              address entry_point,
 136                              Register arg_1,
 137                              Register arg_2,
 138                              bool check_exceptions) {
 139   assert(arg_1 != c_rarg2, "smashed arg");
 140   pass_arg2(this, arg_2);
 141   pass_arg1(this, arg_1);
 142   call_VM_helper(oop_result, entry_point, 2, check_exceptions);
 143 }
 144 
 145 void MacroAssembler::call_VM(Register oop_result,
 146                              address entry_point,
 147                              Register arg_1,
 148                              Register arg_2,
 149                              Register arg_3,
 150                              bool check_exceptions) {
 151   assert(arg_1 != c_rarg3, "smashed arg");
 152   assert(arg_2 != c_rarg3, "smashed arg");
 153   pass_arg3(this, arg_3);
 154 
 155   assert(arg_1 != c_rarg2, "smashed arg");
 156   pass_arg2(this, arg_2);
 157 
 158   pass_arg1(this, arg_1);
 159   call_VM_helper(oop_result, entry_point, 3, check_exceptions);
 160 }
 161 
 162 void MacroAssembler::call_VM(Register oop_result,
 163                              Register last_java_sp,
 164                              address entry_point,
 165                              int number_of_arguments,
 166                              bool check_exceptions) {
 167   call_VM_base(oop_result, xthread, last_java_sp, entry_point, number_of_arguments, check_exceptions);
 168 }
 169 
 170 void MacroAssembler::call_VM(Register oop_result,
 171                              Register last_java_sp,
 172                              address entry_point,
 173                              Register arg_1,
 174                              bool check_exceptions) {
 175   pass_arg1(this, arg_1);
 176   call_VM(oop_result, last_java_sp, entry_point, 1, check_exceptions);
 177 }
 178 
 179 void MacroAssembler::call_VM(Register oop_result,
 180                              Register last_java_sp,
 181                              address entry_point,
 182                              Register arg_1,
 183                              Register arg_2,
 184                              bool check_exceptions) {
 185 
 186   assert(arg_1 != c_rarg2, "smashed arg");
 187   pass_arg2(this, arg_2);
 188   pass_arg1(this, arg_1);
 189   call_VM(oop_result, last_java_sp, entry_point, 2, check_exceptions);
 190 }
 191 
 192 void MacroAssembler::call_VM(Register oop_result,
 193                              Register last_java_sp,
 194                              address entry_point,
 195                              Register arg_1,
 196                              Register arg_2,
 197                              Register arg_3,
 198                              bool check_exceptions) {
 199   assert(arg_1 != c_rarg3, "smashed arg");
 200   assert(arg_2 != c_rarg3, "smashed arg");
 201   pass_arg3(this, arg_3);
 202   assert(arg_1 != c_rarg2, "smashed arg");
 203   pass_arg2(this, arg_2);
 204   pass_arg1(this, arg_1);
 205   call_VM(oop_result, last_java_sp, entry_point, 3, check_exceptions);
 206 }
 207 
 208 void MacroAssembler::post_call_nop() {
 209   if (!Continuations::enabled()) {
 210     return;
 211   }
 212   relocate(post_call_nop_Relocation::spec(), [&] {
 213     InlineSkippedInstructionsCounter skipCounter(this);
 214     nop();
 215     li32(zr, 0);
 216   });
 217 }
 218 
 219 // these are no-ops overridden by InterpreterMacroAssembler
 220 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {}
 221 void MacroAssembler::check_and_handle_popframe(Register java_thread) {}
 222 
 223 // Calls to C land
 224 //
 225 // When entering C land, the fp, & esp of the last Java frame have to be recorded
 226 // in the (thread-local) JavaThread object. When leaving C land, the last Java fp
 227 // has to be reset to 0. This is required to allow proper stack traversal.
 228 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 229                                          Register last_java_fp,
 230                                          Register last_java_pc,
 231                                          Register tmp) {
 232 
 233   if (last_java_pc->is_valid()) {
 234       sd(last_java_pc, Address(xthread,
 235                                JavaThread::frame_anchor_offset() +
 236                                JavaFrameAnchor::last_Java_pc_offset()));
 237   }
 238 
 239   // determine last_java_sp register
 240   if (last_java_sp == sp) {
 241     mv(tmp, sp);
 242     last_java_sp = tmp;
 243   } else if (!last_java_sp->is_valid()) {
 244     last_java_sp = esp;
 245   }
 246 
 247   sd(last_java_sp, Address(xthread, JavaThread::last_Java_sp_offset()));
 248 
 249   // last_java_fp is optional
 250   if (last_java_fp->is_valid()) {
 251     sd(last_java_fp, Address(xthread, JavaThread::last_Java_fp_offset()));
 252   }
 253 }
 254 
 255 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 256                                          Register last_java_fp,
 257                                          address  last_java_pc,
 258                                          Register tmp) {
 259   assert(last_java_pc != nullptr, "must provide a valid PC");
 260 
 261   la(tmp, last_java_pc);
 262   sd(tmp, Address(xthread, JavaThread::frame_anchor_offset() + JavaFrameAnchor::last_Java_pc_offset()));
 263 
 264   set_last_Java_frame(last_java_sp, last_java_fp, noreg, tmp);
 265 }
 266 
 267 void MacroAssembler::set_last_Java_frame(Register last_java_sp,
 268                                          Register last_java_fp,
 269                                          Label &L,
 270                                          Register tmp) {
 271   if (L.is_bound()) {
 272     set_last_Java_frame(last_java_sp, last_java_fp, target(L), tmp);
 273   } else {
 274     L.add_patch_at(code(), locator());
 275     IncompressibleRegion ir(this);  // the label address will be patched back.
 276     set_last_Java_frame(last_java_sp, last_java_fp, pc() /* Patched later */, tmp);
 277   }
 278 }
 279 
 280 void MacroAssembler::reset_last_Java_frame(bool clear_fp) {
 281   // we must set sp to zero to clear frame
 282   sd(zr, Address(xthread, JavaThread::last_Java_sp_offset()));
 283 
 284   // must clear fp, so that compiled frames are not confused; it is
 285   // possible that we need it only for debugging
 286   if (clear_fp) {
 287     sd(zr, Address(xthread, JavaThread::last_Java_fp_offset()));
 288   }
 289 
 290   // Always clear the pc because it could have been set by make_walkable()
 291   sd(zr, Address(xthread, JavaThread::last_Java_pc_offset()));
 292 }
 293 
 294 void MacroAssembler::call_VM_base(Register oop_result,
 295                                   Register java_thread,
 296                                   Register last_java_sp,
 297                                   address  entry_point,
 298                                   int      number_of_arguments,
 299                                   bool     check_exceptions) {
 300    // determine java_thread register
 301   if (!java_thread->is_valid()) {
 302     java_thread = xthread;
 303   }
 304   // determine last_java_sp register
 305   if (!last_java_sp->is_valid()) {
 306     last_java_sp = esp;
 307   }
 308 
 309   // debugging support
 310   assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 311   assert(java_thread == xthread, "unexpected register");
 312 
 313   assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 314   assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 315 
 316   // push java thread (becomes first argument of C function)
 317   mv(c_rarg0, java_thread);
 318 
 319   // set last Java frame before call
 320   assert(last_java_sp != fp, "can't use fp");
 321 
 322   Label l;
 323   set_last_Java_frame(last_java_sp, fp, l, t0);
 324 
 325   // do the call, remove parameters
 326   MacroAssembler::call_VM_leaf_base(entry_point, number_of_arguments, &l);
 327 
 328   // reset last Java frame
 329   // Only interpreter should have to clear fp
 330   reset_last_Java_frame(true);
 331 
 332    // C++ interp handles this in the interpreter
 333   check_and_handle_popframe(java_thread);
 334   check_and_handle_earlyret(java_thread);
 335 
 336   if (check_exceptions) {
 337     // check for pending exceptions (java_thread is set upon return)
 338     ld(t0, Address(java_thread, in_bytes(Thread::pending_exception_offset())));
 339     Label ok;
 340     beqz(t0, ok);
 341     RuntimeAddress target(StubRoutines::forward_exception_entry());
 342     relocate(target.rspec(), [&] {
 343       int32_t offset;
 344       la_patchable(t0, target, offset);
 345       jalr(x0, t0, offset);
 346     });
 347     bind(ok);
 348   }
 349 
 350   // get oop result if there is one and reset the value in the thread
 351   if (oop_result->is_valid()) {
 352     get_vm_result(oop_result, java_thread);
 353   }
 354 }
 355 
 356 void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
 357   ld(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
 358   sd(zr, Address(java_thread, JavaThread::vm_result_offset()));
 359   verify_oop_msg(oop_result, "broken oop in call_VM_base");
 360 }
 361 
 362 void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
 363   ld(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
 364   sd(zr, Address(java_thread, JavaThread::vm_result_2_offset()));
 365 }
 366 
 367 void MacroAssembler::clinit_barrier(Register klass, Register tmp, Label* L_fast_path, Label* L_slow_path) {
 368   assert(L_fast_path != nullptr || L_slow_path != nullptr, "at least one is required");
 369   assert_different_registers(klass, xthread, tmp);
 370 
 371   Label L_fallthrough, L_tmp;
 372   if (L_fast_path == nullptr) {
 373     L_fast_path = &L_fallthrough;
 374   } else if (L_slow_path == nullptr) {
 375     L_slow_path = &L_fallthrough;
 376   }
 377 
 378   // Fast path check: class is fully initialized
 379   lbu(tmp, Address(klass, InstanceKlass::init_state_offset()));
 380   sub(tmp, tmp, InstanceKlass::fully_initialized);
 381   beqz(tmp, *L_fast_path);
 382 
 383   // Fast path check: current thread is initializer thread
 384   ld(tmp, Address(klass, InstanceKlass::init_thread_offset()));
 385 
 386   if (L_slow_path == &L_fallthrough) {
 387     beq(xthread, tmp, *L_fast_path);
 388     bind(*L_slow_path);
 389   } else if (L_fast_path == &L_fallthrough) {
 390     bne(xthread, tmp, *L_slow_path);
 391     bind(*L_fast_path);
 392   } else {
 393     Unimplemented();
 394   }
 395 }
 396 
 397 void MacroAssembler::_verify_oop(Register reg, const char* s, const char* file, int line) {
 398   if (!VerifyOops) { return; }
 399 
 400   // Pass register number to verify_oop_subroutine
 401   const char* b = nullptr;
 402   {
 403     ResourceMark rm;
 404     stringStream ss;
 405     ss.print("verify_oop: %s: %s (%s:%d)", reg->name(), s, file, line);
 406     b = code_string(ss.as_string());
 407   }
 408   BLOCK_COMMENT("verify_oop {");
 409 
 410   push_reg(RegSet::of(ra, t0, t1, c_rarg0), sp);
 411 
 412   mv(c_rarg0, reg); // c_rarg0 : x10
 413   {
 414     // The length of the instruction sequence emitted should not depend
 415     // on the address of the char buffer so that the size of mach nodes for
 416     // scratch emit and normal emit matches.
 417     IncompressibleRegion ir(this);  // Fixed length
 418     movptr(t0, (address) b);
 419   }
 420 
 421   // call indirectly to solve generation ordering problem
 422   ExternalAddress target(StubRoutines::verify_oop_subroutine_entry_address());
 423   relocate(target.rspec(), [&] {
 424     int32_t offset;
 425     la_patchable(t1, target, offset);
 426     ld(t1, Address(t1, offset));
 427   });
 428   jalr(t1);
 429 
 430   pop_reg(RegSet::of(ra, t0, t1, c_rarg0), sp);
 431 
 432   BLOCK_COMMENT("} verify_oop");
 433 }
 434 
 435 void MacroAssembler::_verify_oop_addr(Address addr, const char* s, const char* file, int line) {
 436   if (!VerifyOops) {
 437     return;
 438   }
 439 
 440   const char* b = nullptr;
 441   {
 442     ResourceMark rm;
 443     stringStream ss;
 444     ss.print("verify_oop_addr: %s (%s:%d)", s, file, line);
 445     b = code_string(ss.as_string());
 446   }
 447   BLOCK_COMMENT("verify_oop_addr {");
 448 
 449   push_reg(RegSet::of(ra, t0, t1, c_rarg0), sp);
 450 
 451   if (addr.uses(sp)) {
 452     la(x10, addr);
 453     ld(x10, Address(x10, 4 * wordSize));
 454   } else {
 455     ld(x10, addr);
 456   }
 457 
 458   {
 459     // The length of the instruction sequence emitted should not depend
 460     // on the address of the char buffer so that the size of mach nodes for
 461     // scratch emit and normal emit matches.
 462     IncompressibleRegion ir(this);  // Fixed length
 463     movptr(t0, (address) b);
 464   }
 465 
 466   // call indirectly to solve generation ordering problem
 467   ExternalAddress target(StubRoutines::verify_oop_subroutine_entry_address());
 468   relocate(target.rspec(), [&] {
 469     int32_t offset;
 470     la_patchable(t1, target, offset);
 471     ld(t1, Address(t1, offset));
 472   });
 473   jalr(t1);
 474 
 475   pop_reg(RegSet::of(ra, t0, t1, c_rarg0), sp);
 476 
 477   BLOCK_COMMENT("} verify_oop_addr");
 478 }
 479 
 480 Address MacroAssembler::argument_address(RegisterOrConstant arg_slot,
 481                                          int extra_slot_offset) {
 482   // cf. TemplateTable::prepare_invoke(), if (load_receiver).
 483   int stackElementSize = Interpreter::stackElementSize;
 484   int offset = Interpreter::expr_offset_in_bytes(extra_slot_offset+0);
 485 #ifdef ASSERT
 486   int offset1 = Interpreter::expr_offset_in_bytes(extra_slot_offset+1);
 487   assert(offset1 - offset == stackElementSize, "correct arithmetic");
 488 #endif
 489   if (arg_slot.is_constant()) {
 490     return Address(esp, arg_slot.as_constant() * stackElementSize + offset);
 491   } else {
 492     assert_different_registers(t0, arg_slot.as_register());
 493     shadd(t0, arg_slot.as_register(), esp, t0, exact_log2(stackElementSize));
 494     return Address(t0, offset);
 495   }
 496 }
 497 
 498 #ifndef PRODUCT
 499 extern "C" void findpc(intptr_t x);
 500 #endif
 501 
 502 void MacroAssembler::debug64(char* msg, int64_t pc, int64_t regs[])
 503 {
 504   // In order to get locks to work, we need to fake a in_VM state
 505   if (ShowMessageBoxOnError) {
 506     JavaThread* thread = JavaThread::current();
 507     JavaThreadState saved_state = thread->thread_state();
 508     thread->set_thread_state(_thread_in_vm);
 509 #ifndef PRODUCT
 510     if (CountBytecodes || TraceBytecodes || StopInterpreterAt) {
 511       ttyLocker ttyl;
 512       BytecodeCounter::print();
 513     }
 514 #endif
 515     if (os::message_box(msg, "Execution stopped, print registers?")) {
 516       ttyLocker ttyl;
 517       tty->print_cr(" pc = 0x%016lx", pc);
 518 #ifndef PRODUCT
 519       tty->cr();
 520       findpc(pc);
 521       tty->cr();
 522 #endif
 523       tty->print_cr(" x0 = 0x%016lx", regs[0]);
 524       tty->print_cr(" x1 = 0x%016lx", regs[1]);
 525       tty->print_cr(" x2 = 0x%016lx", regs[2]);
 526       tty->print_cr(" x3 = 0x%016lx", regs[3]);
 527       tty->print_cr(" x4 = 0x%016lx", regs[4]);
 528       tty->print_cr(" x5 = 0x%016lx", regs[5]);
 529       tty->print_cr(" x6 = 0x%016lx", regs[6]);
 530       tty->print_cr(" x7 = 0x%016lx", regs[7]);
 531       tty->print_cr(" x8 = 0x%016lx", regs[8]);
 532       tty->print_cr(" x9 = 0x%016lx", regs[9]);
 533       tty->print_cr("x10 = 0x%016lx", regs[10]);
 534       tty->print_cr("x11 = 0x%016lx", regs[11]);
 535       tty->print_cr("x12 = 0x%016lx", regs[12]);
 536       tty->print_cr("x13 = 0x%016lx", regs[13]);
 537       tty->print_cr("x14 = 0x%016lx", regs[14]);
 538       tty->print_cr("x15 = 0x%016lx", regs[15]);
 539       tty->print_cr("x16 = 0x%016lx", regs[16]);
 540       tty->print_cr("x17 = 0x%016lx", regs[17]);
 541       tty->print_cr("x18 = 0x%016lx", regs[18]);
 542       tty->print_cr("x19 = 0x%016lx", regs[19]);
 543       tty->print_cr("x20 = 0x%016lx", regs[20]);
 544       tty->print_cr("x21 = 0x%016lx", regs[21]);
 545       tty->print_cr("x22 = 0x%016lx", regs[22]);
 546       tty->print_cr("x23 = 0x%016lx", regs[23]);
 547       tty->print_cr("x24 = 0x%016lx", regs[24]);
 548       tty->print_cr("x25 = 0x%016lx", regs[25]);
 549       tty->print_cr("x26 = 0x%016lx", regs[26]);
 550       tty->print_cr("x27 = 0x%016lx", regs[27]);
 551       tty->print_cr("x28 = 0x%016lx", regs[28]);
 552       tty->print_cr("x30 = 0x%016lx", regs[30]);
 553       tty->print_cr("x31 = 0x%016lx", regs[31]);
 554       BREAKPOINT;
 555     }
 556   }
 557   fatal("DEBUG MESSAGE: %s", msg);
 558 }
 559 
 560 void MacroAssembler::resolve_jobject(Register value, Register tmp1, Register tmp2) {
 561   assert_different_registers(value, tmp1, tmp2);
 562   Label done, tagged, weak_tagged;
 563 
 564   beqz(value, done);           // Use null as-is.
 565   // Test for tag.
 566   andi(tmp1, value, JNIHandles::tag_mask);
 567   bnez(tmp1, tagged);
 568 
 569   // Resolve local handle
 570   access_load_at(T_OBJECT, IN_NATIVE | AS_RAW, value, Address(value, 0), tmp1, tmp2);
 571   verify_oop(value);
 572   j(done);
 573 
 574   bind(tagged);
 575   // Test for jweak tag.
 576   STATIC_ASSERT(JNIHandles::TypeTag::weak_global == 0b1);
 577   test_bit(tmp1, value, exact_log2(JNIHandles::TypeTag::weak_global));
 578   bnez(tmp1, weak_tagged);
 579 
 580   // Resolve global handle
 581   access_load_at(T_OBJECT, IN_NATIVE, value,
 582                  Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2);
 583   verify_oop(value);
 584   j(done);
 585 
 586   bind(weak_tagged);
 587   // Resolve jweak.
 588   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF, value,
 589                  Address(value, -JNIHandles::TypeTag::weak_global), tmp1, tmp2);
 590   verify_oop(value);
 591 
 592   bind(done);
 593 }
 594 
 595 void MacroAssembler::resolve_global_jobject(Register value, Register tmp1, Register tmp2) {
 596   assert_different_registers(value, tmp1, tmp2);
 597   Label done;
 598 
 599   beqz(value, done);           // Use null as-is.
 600 
 601 #ifdef ASSERT
 602   {
 603     STATIC_ASSERT(JNIHandles::TypeTag::global == 0b10);
 604     Label valid_global_tag;
 605     test_bit(tmp1, value, exact_log2(JNIHandles::TypeTag::global)); // Test for global tag.
 606     bnez(tmp1, valid_global_tag);
 607     stop("non global jobject using resolve_global_jobject");
 608     bind(valid_global_tag);
 609   }
 610 #endif
 611 
 612   // Resolve global handle
 613   access_load_at(T_OBJECT, IN_NATIVE, value,
 614                  Address(value, -JNIHandles::TypeTag::global), tmp1, tmp2);
 615   verify_oop(value);
 616 
 617   bind(done);
 618 }
 619 
 620 void MacroAssembler::stop(const char* msg) {
 621   BLOCK_COMMENT(msg);
 622   illegal_instruction(Assembler::csr::time);
 623   emit_int64((uintptr_t)msg);
 624 }
 625 
 626 void MacroAssembler::unimplemented(const char* what) {
 627   const char* buf = nullptr;
 628   {
 629     ResourceMark rm;
 630     stringStream ss;
 631     ss.print("unimplemented: %s", what);
 632     buf = code_string(ss.as_string());
 633   }
 634   stop(buf);
 635 }
 636 
 637 void MacroAssembler::emit_static_call_stub() {
 638   IncompressibleRegion ir(this);  // Fixed length: see CompiledStaticCall::to_interp_stub_size().
 639   // CompiledDirectStaticCall::set_to_interpreted knows the
 640   // exact layout of this stub.
 641 
 642   mov_metadata(xmethod, (Metadata*)nullptr);
 643 
 644   // Jump to the entry point of the c2i stub.
 645   int32_t offset = 0;
 646   movptr(t0, 0, offset);
 647   jalr(x0, t0, offset);
 648 }
 649 
 650 void MacroAssembler::call_VM_leaf_base(address entry_point,
 651                                        int number_of_arguments,
 652                                        Label *retaddr) {
 653   push_reg(RegSet::of(t0, xmethod), sp);   // push << t0 & xmethod >> to sp
 654   call(entry_point);
 655   if (retaddr != nullptr) {
 656     bind(*retaddr);
 657   }
 658   pop_reg(RegSet::of(t0, xmethod), sp);   // pop << t0 & xmethod >> from sp
 659 }
 660 
 661 void MacroAssembler::call_VM_leaf(address entry_point, int number_of_arguments) {
 662   call_VM_leaf_base(entry_point, number_of_arguments);
 663 }
 664 
 665 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0) {
 666   pass_arg0(this, arg_0);
 667   call_VM_leaf_base(entry_point, 1);
 668 }
 669 
 670 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 671   pass_arg0(this, arg_0);
 672   pass_arg1(this, arg_1);
 673   call_VM_leaf_base(entry_point, 2);
 674 }
 675 
 676 void MacroAssembler::call_VM_leaf(address entry_point, Register arg_0,
 677                                   Register arg_1, Register arg_2) {
 678   pass_arg0(this, arg_0);
 679   pass_arg1(this, arg_1);
 680   pass_arg2(this, arg_2);
 681   call_VM_leaf_base(entry_point, 3);
 682 }
 683 
 684 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0) {
 685   pass_arg0(this, arg_0);
 686   MacroAssembler::call_VM_leaf_base(entry_point, 1);
 687 }
 688 
 689 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1) {
 690 
 691   assert(arg_0 != c_rarg1, "smashed arg");
 692   pass_arg1(this, arg_1);
 693   pass_arg0(this, arg_0);
 694   MacroAssembler::call_VM_leaf_base(entry_point, 2);
 695 }
 696 
 697 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2) {
 698   assert(arg_0 != c_rarg2, "smashed arg");
 699   assert(arg_1 != c_rarg2, "smashed arg");
 700   pass_arg2(this, arg_2);
 701   assert(arg_0 != c_rarg1, "smashed arg");
 702   pass_arg1(this, arg_1);
 703   pass_arg0(this, arg_0);
 704   MacroAssembler::call_VM_leaf_base(entry_point, 3);
 705 }
 706 
 707 void MacroAssembler::super_call_VM_leaf(address entry_point, Register arg_0, Register arg_1, Register arg_2, Register arg_3) {
 708   assert(arg_0 != c_rarg3, "smashed arg");
 709   assert(arg_1 != c_rarg3, "smashed arg");
 710   assert(arg_2 != c_rarg3, "smashed arg");
 711   pass_arg3(this, arg_3);
 712   assert(arg_0 != c_rarg2, "smashed arg");
 713   assert(arg_1 != c_rarg2, "smashed arg");
 714   pass_arg2(this, arg_2);
 715   assert(arg_0 != c_rarg1, "smashed arg");
 716   pass_arg1(this, arg_1);
 717   pass_arg0(this, arg_0);
 718   MacroAssembler::call_VM_leaf_base(entry_point, 4);
 719 }
 720 
 721 void MacroAssembler::la(Register Rd, const address dest) {
 722   int64_t offset = dest - pc();
 723   if (is_simm32(offset)) {
 724     auipc(Rd, (int32_t)offset + 0x800);  //0x800, Note:the 11th sign bit
 725     addi(Rd, Rd, ((int64_t)offset << 52) >> 52);
 726   } else {
 727     movptr(Rd, dest);
 728   }
 729 }
 730 
 731 void MacroAssembler::la(Register Rd, const Address &adr) {
 732   switch (adr.getMode()) {
 733     case Address::literal: {
 734       relocInfo::relocType rtype = adr.rspec().reloc()->type();
 735       if (rtype == relocInfo::none) {
 736         mv(Rd, (intptr_t)(adr.target()));
 737       } else {
 738         relocate(adr.rspec(), [&] {
 739           movptr(Rd, adr.target());
 740         });
 741       }
 742       break;
 743     }
 744     case Address::base_plus_offset: {
 745       Address new_adr = legitimize_address(Rd, adr);
 746       if (!(new_adr.base() == Rd && new_adr.offset() == 0)) {
 747         addi(Rd, new_adr.base(), new_adr.offset());
 748       }
 749       break;
 750     }
 751     default:
 752       ShouldNotReachHere();
 753   }
 754 }
 755 
 756 void MacroAssembler::la(Register Rd, Label &label) {
 757   IncompressibleRegion ir(this);   // the label address may be patched back.
 758   wrap_label(Rd, label, &MacroAssembler::la);
 759 }
 760 
 761 void MacroAssembler::li16u(Register Rd, uint16_t imm) {
 762   lui(Rd, (uint32_t)imm << 12);
 763   srli(Rd, Rd, 12);
 764 }
 765 
 766 void MacroAssembler::li32(Register Rd, int32_t imm) {
 767   // int32_t is in range 0x8000 0000 ~ 0x7fff ffff, and imm[31] is the sign bit
 768   int64_t upper = imm, lower = imm;
 769   lower = (imm << 20) >> 20;
 770   upper -= lower;
 771   upper = (int32_t)upper;
 772   // lui Rd, imm[31:12] + imm[11]
 773   lui(Rd, upper);
 774   // use addiw to distinguish li32 to li64
 775   addiw(Rd, Rd, lower);
 776 }
 777 
 778 void MacroAssembler::li64(Register Rd, int64_t imm) {
 779   // Load upper 32 bits. upper = imm[63:32], but if imm[31] == 1 or
 780   // (imm[31:20] == 0x7ff && imm[19] == 1), upper = imm[63:32] + 1.
 781   int64_t lower = imm & 0xffffffff;
 782   lower -= ((lower << 44) >> 44);
 783   int64_t tmp_imm = ((uint64_t)(imm & 0xffffffff00000000)) + (uint64_t)lower;
 784   int32_t upper = (tmp_imm - (int32_t)lower) >> 32;
 785 
 786   // Load upper 32 bits
 787   int64_t up = upper, lo = upper;
 788   lo = (lo << 52) >> 52;
 789   up -= lo;
 790   up = (int32_t)up;
 791   lui(Rd, up);
 792   addi(Rd, Rd, lo);
 793 
 794   // Load the rest 32 bits.
 795   slli(Rd, Rd, 12);
 796   addi(Rd, Rd, (int32_t)lower >> 20);
 797   slli(Rd, Rd, 12);
 798   lower = ((int32_t)imm << 12) >> 20;
 799   addi(Rd, Rd, lower);
 800   slli(Rd, Rd, 8);
 801   lower = imm & 0xff;
 802   addi(Rd, Rd, lower);
 803 }
 804 
 805 void MacroAssembler::li(Register Rd, int64_t imm) {
 806   // int64_t is in range 0x8000 0000 0000 0000 ~ 0x7fff ffff ffff ffff
 807   // li -> c.li
 808   if (do_compress() && (is_simm6(imm) && Rd != x0)) {
 809     c_li(Rd, imm);
 810     return;
 811   }
 812 
 813   int shift = 12;
 814   int64_t upper = imm, lower = imm;
 815   // Split imm to a lower 12-bit sign-extended part and the remainder,
 816   // because addi will sign-extend the lower imm.
 817   lower = ((int32_t)imm << 20) >> 20;
 818   upper -= lower;
 819 
 820   // Test whether imm is a 32-bit integer.
 821   if (!(((imm) & ~(int64_t)0x7fffffff) == 0 ||
 822         (((imm) & ~(int64_t)0x7fffffff) == ~(int64_t)0x7fffffff))) {
 823     while (((upper >> shift) & 1) == 0) { shift++; }
 824     upper >>= shift;
 825     li(Rd, upper);
 826     slli(Rd, Rd, shift);
 827     if (lower != 0) {
 828       addi(Rd, Rd, lower);
 829     }
 830   } else {
 831     // 32-bit integer
 832     Register hi_Rd = zr;
 833     if (upper != 0) {
 834       lui(Rd, (int32_t)upper);
 835       hi_Rd = Rd;
 836     }
 837     if (lower != 0 || hi_Rd == zr) {
 838       addiw(Rd, hi_Rd, lower);
 839     }
 840   }
 841 }
 842 
 843 #define INSN(NAME, REGISTER)                                       \
 844   void MacroAssembler::NAME(const address dest, Register temp) {   \
 845     assert_cond(dest != nullptr);                                  \
 846     int64_t distance = dest - pc();                                \
 847     if (is_simm21(distance) && ((distance % 2) == 0)) {            \
 848       Assembler::jal(REGISTER, distance);                          \
 849     } else {                                                       \
 850       assert(temp != noreg, "expecting a register");               \
 851       int32_t offset = 0;                                          \
 852       movptr(temp, dest, offset);                                  \
 853       Assembler::jalr(REGISTER, temp, offset);                     \
 854     }                                                              \
 855   }                                                                \
 856 
 857   INSN(j,   x0);
 858   INSN(jal, x1);
 859 
 860 #undef INSN
 861 
 862 #define INSN(NAME, REGISTER)                                       \
 863   void MacroAssembler::NAME(const Address &adr, Register temp) {   \
 864     switch (adr.getMode()) {                                       \
 865       case Address::literal: {                                     \
 866         relocate(adr.rspec(), [&] {                                \
 867           NAME(adr.target(), temp);                                \
 868         });                                                        \
 869         break;                                                     \
 870       }                                                            \
 871       case Address::base_plus_offset: {                            \
 872         int32_t offset = ((int32_t)adr.offset() << 20) >> 20;      \
 873         la(temp, Address(adr.base(), adr.offset() - offset));      \
 874         Assembler::jalr(REGISTER, temp, offset);                   \
 875         break;                                                     \
 876       }                                                            \
 877       default:                                                     \
 878         ShouldNotReachHere();                                      \
 879     }                                                              \
 880   }
 881 
 882   INSN(j,   x0);
 883   INSN(jal, x1);
 884 
 885 #undef INSN
 886 
 887 #define INSN(NAME)                                                                    \
 888   void MacroAssembler::NAME(Register Rd, const address dest, Register temp) {         \
 889     assert_cond(dest != nullptr);                                                     \
 890     int64_t distance = dest - pc();                                                   \
 891     if (is_simm21(distance) && ((distance % 2) == 0)) {                               \
 892       Assembler::NAME(Rd, distance);                                                  \
 893     } else {                                                                          \
 894       assert_different_registers(Rd, temp);                                           \
 895       int32_t offset = 0;                                                             \
 896       movptr(temp, dest, offset);                                                     \
 897       jalr(Rd, temp, offset);                                                         \
 898     }                                                                                 \
 899   }                                                                                   \
 900   void MacroAssembler::NAME(Register Rd, Label &L, Register temp) {                   \
 901     assert_different_registers(Rd, temp);                                             \
 902     wrap_label(Rd, L, temp, &MacroAssembler::NAME);                                   \
 903   }
 904 
 905   INSN(jal);
 906 
 907 #undef INSN
 908 
 909 #define INSN(NAME, REGISTER)                                       \
 910   void MacroAssembler::NAME(Label &l, Register temp) {             \
 911     jal(REGISTER, l, temp);                                        \
 912   }                                                                \
 913 
 914   INSN(j,   x0);
 915   INSN(jal, x1);
 916 
 917 #undef INSN
 918 
 919 void MacroAssembler::wrap_label(Register Rt, Label &L, Register tmp, load_insn_by_temp insn) {
 920   if (L.is_bound()) {
 921     (this->*insn)(Rt, target(L), tmp);
 922   } else {
 923     L.add_patch_at(code(), locator());
 924     (this->*insn)(Rt, pc(), tmp);
 925   }
 926 }
 927 
 928 void MacroAssembler::wrap_label(Register Rt, Label &L, jal_jalr_insn insn) {
 929   if (L.is_bound()) {
 930     (this->*insn)(Rt, target(L));
 931   } else {
 932     L.add_patch_at(code(), locator());
 933     (this->*insn)(Rt, pc());
 934   }
 935 }
 936 
 937 void MacroAssembler::wrap_label(Register r1, Register r2, Label &L,
 938                                 compare_and_branch_insn insn,
 939                                 compare_and_branch_label_insn neg_insn, bool is_far) {
 940   if (is_far) {
 941     Label done;
 942     (this->*neg_insn)(r1, r2, done, /* is_far */ false);
 943     j(L);
 944     bind(done);
 945   } else {
 946     if (L.is_bound()) {
 947       (this->*insn)(r1, r2, target(L));
 948     } else {
 949       L.add_patch_at(code(), locator());
 950       (this->*insn)(r1, r2, pc());
 951     }
 952   }
 953 }
 954 
 955 #define INSN(NAME, NEG_INSN)                                                              \
 956   void MacroAssembler::NAME(Register Rs1, Register Rs2, Label &L, bool is_far) {          \
 957     wrap_label(Rs1, Rs2, L, &MacroAssembler::NAME, &MacroAssembler::NEG_INSN, is_far);    \
 958   }
 959 
 960   INSN(beq,  bne);
 961   INSN(bne,  beq);
 962   INSN(blt,  bge);
 963   INSN(bge,  blt);
 964   INSN(bltu, bgeu);
 965   INSN(bgeu, bltu);
 966 
 967 #undef INSN
 968 
 969 #define INSN(NAME)                                                                \
 970   void MacroAssembler::NAME##z(Register Rs, const address dest) {                 \
 971     NAME(Rs, zr, dest);                                                           \
 972   }                                                                               \
 973   void MacroAssembler::NAME##z(Register Rs, Label &l, bool is_far) {              \
 974     NAME(Rs, zr, l, is_far);                                                      \
 975   }                                                                               \
 976 
 977   INSN(beq);
 978   INSN(bne);
 979   INSN(blt);
 980   INSN(ble);
 981   INSN(bge);
 982   INSN(bgt);
 983 
 984 #undef INSN
 985 
 986 #define INSN(NAME, NEG_INSN)                                                      \
 987   void MacroAssembler::NAME(Register Rs, Register Rt, const address dest) {       \
 988     NEG_INSN(Rt, Rs, dest);                                                       \
 989   }                                                                               \
 990   void MacroAssembler::NAME(Register Rs, Register Rt, Label &l, bool is_far) {    \
 991     NEG_INSN(Rt, Rs, l, is_far);                                                  \
 992   }
 993 
 994   INSN(bgt,  blt);
 995   INSN(ble,  bge);
 996   INSN(bgtu, bltu);
 997   INSN(bleu, bgeu);
 998 
 999 #undef INSN
1000 
1001 // Float compare branch instructions
1002 
1003 #define INSN(NAME, FLOATCMP, BRANCH)                                                                                    \
1004   void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far, bool is_unordered) {   \
1005     FLOATCMP##_s(t0, Rs1, Rs2);                                                                                         \
1006     BRANCH(t0, l, is_far);                                                                                              \
1007   }                                                                                                                     \
1008   void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, bool is_far, bool is_unordered) {  \
1009     FLOATCMP##_d(t0, Rs1, Rs2);                                                                                         \
1010     BRANCH(t0, l, is_far);                                                                                              \
1011   }
1012 
1013   INSN(beq, feq, bnez);
1014   INSN(bne, feq, beqz);
1015 
1016 #undef INSN
1017 
1018 
1019 #define INSN(NAME, FLOATCMP1, FLOATCMP2)                                              \
1020   void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l,   \
1021                                     bool is_far, bool is_unordered) {                 \
1022     if (is_unordered) {                                                               \
1023       /* jump if either source is NaN or condition is expected */                     \
1024       FLOATCMP2##_s(t0, Rs2, Rs1);                                                    \
1025       beqz(t0, l, is_far);                                                            \
1026     } else {                                                                          \
1027       /* jump if no NaN in source and condition is expected */                        \
1028       FLOATCMP1##_s(t0, Rs1, Rs2);                                                    \
1029       bnez(t0, l, is_far);                                                            \
1030     }                                                                                 \
1031   }                                                                                   \
1032   void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l,  \
1033                                      bool is_far, bool is_unordered) {                \
1034     if (is_unordered) {                                                               \
1035       /* jump if either source is NaN or condition is expected */                     \
1036       FLOATCMP2##_d(t0, Rs2, Rs1);                                                    \
1037       beqz(t0, l, is_far);                                                            \
1038     } else {                                                                          \
1039       /* jump if no NaN in source and condition is expected */                        \
1040       FLOATCMP1##_d(t0, Rs1, Rs2);                                                    \
1041       bnez(t0, l, is_far);                                                            \
1042     }                                                                                 \
1043   }
1044 
1045   INSN(ble, fle, flt);
1046   INSN(blt, flt, fle);
1047 
1048 #undef INSN
1049 
1050 #define INSN(NAME, CMP)                                                              \
1051   void MacroAssembler::float_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l,  \
1052                                     bool is_far, bool is_unordered) {                \
1053     float_##CMP(Rs2, Rs1, l, is_far, is_unordered);                                  \
1054   }                                                                                  \
1055   void MacroAssembler::double_##NAME(FloatRegister Rs1, FloatRegister Rs2, Label &l, \
1056                                      bool is_far, bool is_unordered) {               \
1057     double_##CMP(Rs2, Rs1, l, is_far, is_unordered);                                 \
1058   }
1059 
1060   INSN(bgt, blt);
1061   INSN(bge, ble);
1062 
1063 #undef INSN
1064 
1065 
1066 #define INSN(NAME, CSR)                       \
1067   void MacroAssembler::NAME(Register Rd) {    \
1068     csrr(Rd, CSR);                            \
1069   }
1070 
1071   INSN(rdinstret,  CSR_INSTRET);
1072   INSN(rdcycle,    CSR_CYCLE);
1073   INSN(rdtime,     CSR_TIME);
1074   INSN(frcsr,      CSR_FCSR);
1075   INSN(frrm,       CSR_FRM);
1076   INSN(frflags,    CSR_FFLAGS);
1077 
1078 #undef INSN
1079 
1080 void MacroAssembler::csrr(Register Rd, unsigned csr) {
1081   csrrs(Rd, csr, x0);
1082 }
1083 
1084 #define INSN(NAME, OPFUN)                                      \
1085   void MacroAssembler::NAME(unsigned csr, Register Rs) {       \
1086     OPFUN(x0, csr, Rs);                                        \
1087   }
1088 
1089   INSN(csrw, csrrw);
1090   INSN(csrs, csrrs);
1091   INSN(csrc, csrrc);
1092 
1093 #undef INSN
1094 
1095 #define INSN(NAME, OPFUN)                                      \
1096   void MacroAssembler::NAME(unsigned csr, unsigned imm) {      \
1097     OPFUN(x0, csr, imm);                                       \
1098   }
1099 
1100   INSN(csrwi, csrrwi);
1101   INSN(csrsi, csrrsi);
1102   INSN(csrci, csrrci);
1103 
1104 #undef INSN
1105 
1106 #define INSN(NAME, CSR)                                      \
1107   void MacroAssembler::NAME(Register Rd, Register Rs) {      \
1108     csrrw(Rd, CSR, Rs);                                      \
1109   }
1110 
1111   INSN(fscsr,   CSR_FCSR);
1112   INSN(fsrm,    CSR_FRM);
1113   INSN(fsflags, CSR_FFLAGS);
1114 
1115 #undef INSN
1116 
1117 #define INSN(NAME)                              \
1118   void MacroAssembler::NAME(Register Rs) {      \
1119     NAME(x0, Rs);                               \
1120   }
1121 
1122   INSN(fscsr);
1123   INSN(fsrm);
1124   INSN(fsflags);
1125 
1126 #undef INSN
1127 
1128 void MacroAssembler::fsrmi(Register Rd, unsigned imm) {
1129   guarantee(imm < 5, "Rounding Mode is invalid in Rounding Mode register");
1130   csrrwi(Rd, CSR_FRM, imm);
1131 }
1132 
1133 void MacroAssembler::fsflagsi(Register Rd, unsigned imm) {
1134    csrrwi(Rd, CSR_FFLAGS, imm);
1135 }
1136 
1137 #define INSN(NAME)                             \
1138   void MacroAssembler::NAME(unsigned imm) {    \
1139     NAME(x0, imm);                             \
1140   }
1141 
1142   INSN(fsrmi);
1143   INSN(fsflagsi);
1144 
1145 #undef INSN
1146 
1147 void MacroAssembler::push_reg(Register Rs)
1148 {
1149   addi(esp, esp, 0 - wordSize);
1150   sd(Rs, Address(esp, 0));
1151 }
1152 
1153 void MacroAssembler::pop_reg(Register Rd)
1154 {
1155   ld(Rd, Address(esp, 0));
1156   addi(esp, esp, wordSize);
1157 }
1158 
1159 int MacroAssembler::bitset_to_regs(unsigned int bitset, unsigned char* regs) {
1160   int count = 0;
1161   // Scan bitset to accumulate register pairs
1162   for (int reg = 31; reg >= 0; reg--) {
1163     if ((1U << 31) & bitset) {
1164       regs[count++] = reg;
1165     }
1166     bitset <<= 1;
1167   }
1168   return count;
1169 }
1170 
1171 // Push integer registers in the bitset supplied. Don't push sp.
1172 // Return the number of words pushed
1173 int MacroAssembler::push_reg(unsigned int bitset, Register stack) {
1174   DEBUG_ONLY(int words_pushed = 0;)
1175   unsigned char regs[32];
1176   int count = bitset_to_regs(bitset, regs);
1177   // reserve one slot to align for odd count
1178   int offset = is_even(count) ? 0 : wordSize;
1179 
1180   if (count) {
1181     addi(stack, stack, -count * wordSize - offset);
1182   }
1183   for (int i = count - 1; i >= 0; i--) {
1184     sd(as_Register(regs[i]), Address(stack, (count - 1 - i) * wordSize + offset));
1185     DEBUG_ONLY(words_pushed++;)
1186   }
1187 
1188   assert(words_pushed == count, "oops, pushed != count");
1189 
1190   return count;
1191 }
1192 
1193 int MacroAssembler::pop_reg(unsigned int bitset, Register stack) {
1194   DEBUG_ONLY(int words_popped = 0;)
1195   unsigned char regs[32];
1196   int count = bitset_to_regs(bitset, regs);
1197   // reserve one slot to align for odd count
1198   int offset = is_even(count) ? 0 : wordSize;
1199 
1200   for (int i = count - 1; i >= 0; i--) {
1201     ld(as_Register(regs[i]), Address(stack, (count - 1 - i) * wordSize + offset));
1202     DEBUG_ONLY(words_popped++;)
1203   }
1204 
1205   if (count) {
1206     addi(stack, stack, count * wordSize + offset);
1207   }
1208   assert(words_popped == count, "oops, popped != count");
1209 
1210   return count;
1211 }
1212 
1213 // Push floating-point registers in the bitset supplied.
1214 // Return the number of words pushed
1215 int MacroAssembler::push_fp(unsigned int bitset, Register stack) {
1216   DEBUG_ONLY(int words_pushed = 0;)
1217   unsigned char regs[32];
1218   int count = bitset_to_regs(bitset, regs);
1219   int push_slots = count + (count & 1);
1220 
1221   if (count) {
1222     addi(stack, stack, -push_slots * wordSize);
1223   }
1224 
1225   for (int i = count - 1; i >= 0; i--) {
1226     fsd(as_FloatRegister(regs[i]), Address(stack, (push_slots - 1 - i) * wordSize));
1227     DEBUG_ONLY(words_pushed++;)
1228   }
1229 
1230   assert(words_pushed == count, "oops, pushed(%d) != count(%d)", words_pushed, count);
1231 
1232   return count;
1233 }
1234 
1235 int MacroAssembler::pop_fp(unsigned int bitset, Register stack) {
1236   DEBUG_ONLY(int words_popped = 0;)
1237   unsigned char regs[32];
1238   int count = bitset_to_regs(bitset, regs);
1239   int pop_slots = count + (count & 1);
1240 
1241   for (int i = count - 1; i >= 0; i--) {
1242     fld(as_FloatRegister(regs[i]), Address(stack, (pop_slots - 1 - i) * wordSize));
1243     DEBUG_ONLY(words_popped++;)
1244   }
1245 
1246   if (count) {
1247     addi(stack, stack, pop_slots * wordSize);
1248   }
1249 
1250   assert(words_popped == count, "oops, popped(%d) != count(%d)", words_popped, count);
1251 
1252   return count;
1253 }
1254 
1255 #ifdef COMPILER2
1256 // Push vector registers in the bitset supplied.
1257 // Return the number of words pushed
1258 int MacroAssembler::push_v(unsigned int bitset, Register stack) {
1259   int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1260 
1261   // Scan bitset to accumulate register pairs
1262   unsigned char regs[32];
1263   int count = bitset_to_regs(bitset, regs);
1264 
1265   for (int i = 0; i < count; i++) {
1266     sub(stack, stack, vector_size_in_bytes);
1267     vs1r_v(as_VectorRegister(regs[i]), stack);
1268   }
1269 
1270   return count * vector_size_in_bytes / wordSize;
1271 }
1272 
1273 int MacroAssembler::pop_v(unsigned int bitset, Register stack) {
1274   int vector_size_in_bytes = Matcher::scalable_vector_reg_size(T_BYTE);
1275 
1276   // Scan bitset to accumulate register pairs
1277   unsigned char regs[32];
1278   int count = bitset_to_regs(bitset, regs);
1279 
1280   for (int i = count - 1; i >= 0; i--) {
1281     vl1r_v(as_VectorRegister(regs[i]), stack);
1282     add(stack, stack, vector_size_in_bytes);
1283   }
1284 
1285   return count * vector_size_in_bytes / wordSize;
1286 }
1287 #endif // COMPILER2
1288 
1289 void MacroAssembler::push_call_clobbered_registers_except(RegSet exclude) {
1290   // Push integer registers x7, x10-x17, x28-x31.
1291   push_reg(RegSet::of(x7) + RegSet::range(x10, x17) + RegSet::range(x28, x31) - exclude, sp);
1292 
1293   // Push float registers f0-f7, f10-f17, f28-f31.
1294   addi(sp, sp, - wordSize * 20);
1295   int offset = 0;
1296   for (int i = 0; i < 32; i++) {
1297     if (i <= f7->encoding() || i >= f28->encoding() || (i >= f10->encoding() && i <= f17->encoding())) {
1298       fsd(as_FloatRegister(i), Address(sp, wordSize * (offset++)));
1299     }
1300   }
1301 }
1302 
1303 void MacroAssembler::pop_call_clobbered_registers_except(RegSet exclude) {
1304   int offset = 0;
1305   for (int i = 0; i < 32; i++) {
1306     if (i <= f7->encoding() || i >= f28->encoding() || (i >= f10->encoding() && i <= f17->encoding())) {
1307       fld(as_FloatRegister(i), Address(sp, wordSize * (offset++)));
1308     }
1309   }
1310   addi(sp, sp, wordSize * 20);
1311 
1312   pop_reg(RegSet::of(x7) + RegSet::range(x10, x17) + RegSet::range(x28, x31) - exclude, sp);
1313 }
1314 
1315 void MacroAssembler::push_CPU_state(bool save_vectors, int vector_size_in_bytes) {
1316   // integer registers, except zr(x0) & ra(x1) & sp(x2) & gp(x3) & tp(x4)
1317   push_reg(RegSet::range(x5, x31), sp);
1318 
1319   // float registers
1320   addi(sp, sp, - 32 * wordSize);
1321   for (int i = 0; i < 32; i++) {
1322     fsd(as_FloatRegister(i), Address(sp, i * wordSize));
1323   }
1324 
1325   // vector registers
1326   if (save_vectors) {
1327     sub(sp, sp, vector_size_in_bytes * VectorRegister::number_of_registers);
1328     vsetvli(t0, x0, Assembler::e64, Assembler::m8);
1329     for (int i = 0; i < VectorRegister::number_of_registers; i += 8) {
1330       add(t0, sp, vector_size_in_bytes * i);
1331       vse64_v(as_VectorRegister(i), t0);
1332     }
1333   }
1334 }
1335 
1336 void MacroAssembler::pop_CPU_state(bool restore_vectors, int vector_size_in_bytes) {
1337   // vector registers
1338   if (restore_vectors) {
1339     vsetvli(t0, x0, Assembler::e64, Assembler::m8);
1340     for (int i = 0; i < VectorRegister::number_of_registers; i += 8) {
1341       vle64_v(as_VectorRegister(i), sp);
1342       add(sp, sp, vector_size_in_bytes * 8);
1343     }
1344   }
1345 
1346   // float registers
1347   for (int i = 0; i < 32; i++) {
1348     fld(as_FloatRegister(i), Address(sp, i * wordSize));
1349   }
1350   addi(sp, sp, 32 * wordSize);
1351 
1352   // integer registers, except zr(x0) & ra(x1) & sp(x2) & gp(x3) & tp(x4)
1353   pop_reg(RegSet::range(x5, x31), sp);
1354 }
1355 
1356 static int patch_offset_in_jal(address branch, int64_t offset) {
1357   assert(Assembler::is_simm21(offset) && ((offset % 2) == 0),
1358          "offset is too large to be patched in one jal instruction!\n");
1359   Assembler::patch(branch, 31, 31, (offset >> 20) & 0x1);                       // offset[20]    ==> branch[31]
1360   Assembler::patch(branch, 30, 21, (offset >> 1)  & 0x3ff);                     // offset[10:1]  ==> branch[30:21]
1361   Assembler::patch(branch, 20, 20, (offset >> 11) & 0x1);                       // offset[11]    ==> branch[20]
1362   Assembler::patch(branch, 19, 12, (offset >> 12) & 0xff);                      // offset[19:12] ==> branch[19:12]
1363   return NativeInstruction::instruction_size;                                   // only one instruction
1364 }
1365 
1366 static int patch_offset_in_conditional_branch(address branch, int64_t offset) {
1367   assert(Assembler::is_simm13(offset) && ((offset % 2) == 0),
1368          "offset is too large to be patched in one beq/bge/bgeu/blt/bltu/bne instruction!\n");
1369   Assembler::patch(branch, 31, 31, (offset >> 12) & 0x1);                       // offset[12]    ==> branch[31]
1370   Assembler::patch(branch, 30, 25, (offset >> 5)  & 0x3f);                      // offset[10:5]  ==> branch[30:25]
1371   Assembler::patch(branch, 7,  7,  (offset >> 11) & 0x1);                       // offset[11]    ==> branch[7]
1372   Assembler::patch(branch, 11, 8,  (offset >> 1)  & 0xf);                       // offset[4:1]   ==> branch[11:8]
1373   return NativeInstruction::instruction_size;                                   // only one instruction
1374 }
1375 
1376 static int patch_offset_in_pc_relative(address branch, int64_t offset) {
1377   const int PC_RELATIVE_INSTRUCTION_NUM = 2;                                    // auipc, addi/jalr/load
1378   Assembler::patch(branch, 31, 12, ((offset + 0x800) >> 12) & 0xfffff);         // Auipc.          offset[31:12]  ==> branch[31:12]
1379   Assembler::patch(branch + 4, 31, 20, offset & 0xfff);                         // Addi/Jalr/Load. offset[11:0]   ==> branch[31:20]
1380   return PC_RELATIVE_INSTRUCTION_NUM * NativeInstruction::instruction_size;
1381 }
1382 
1383 static int patch_addr_in_movptr(address branch, address target) {
1384   const int MOVPTR_INSTRUCTIONS_NUM = 6;                                        // lui + addi + slli + addi + slli + addi/jalr/load
1385   int32_t lower = ((intptr_t)target << 35) >> 35;
1386   int64_t upper = ((intptr_t)target - lower) >> 29;
1387   Assembler::patch(branch + 0,  31, 12, upper & 0xfffff);                       // Lui.             target[48:29] + target[28] ==> branch[31:12]
1388   Assembler::patch(branch + 4,  31, 20, (lower >> 17) & 0xfff);                 // Addi.            target[28:17] ==> branch[31:20]
1389   Assembler::patch(branch + 12, 31, 20, (lower >> 6) & 0x7ff);                  // Addi.            target[16: 6] ==> branch[31:20]
1390   Assembler::patch(branch + 20, 31, 20, lower & 0x3f);                          // Addi/Jalr/Load.  target[ 5: 0] ==> branch[31:20]
1391   return MOVPTR_INSTRUCTIONS_NUM * NativeInstruction::instruction_size;
1392 }
1393 
1394 static int patch_imm_in_li64(address branch, address target) {
1395   const int LI64_INSTRUCTIONS_NUM = 8;                                          // lui + addi + slli + addi + slli + addi + slli + addi
1396   int64_t lower = (intptr_t)target & 0xffffffff;
1397   lower = lower - ((lower << 44) >> 44);
1398   int64_t tmp_imm = ((uint64_t)((intptr_t)target & 0xffffffff00000000)) + (uint64_t)lower;
1399   int32_t upper =  (tmp_imm - (int32_t)lower) >> 32;
1400   int64_t tmp_upper = upper, tmp_lower = upper;
1401   tmp_lower = (tmp_lower << 52) >> 52;
1402   tmp_upper -= tmp_lower;
1403   tmp_upper >>= 12;
1404   // Load upper 32 bits. Upper = target[63:32], but if target[31] = 1 or (target[31:20] == 0x7ff && target[19] == 1),
1405   // upper = target[63:32] + 1.
1406   Assembler::patch(branch + 0,  31, 12, tmp_upper & 0xfffff);                       // Lui.
1407   Assembler::patch(branch + 4,  31, 20, tmp_lower & 0xfff);                         // Addi.
1408   // Load the rest 32 bits.
1409   Assembler::patch(branch + 12, 31, 20, ((int32_t)lower >> 20) & 0xfff);            // Addi.
1410   Assembler::patch(branch + 20, 31, 20, (((intptr_t)target << 44) >> 52) & 0xfff);  // Addi.
1411   Assembler::patch(branch + 28, 31, 20, (intptr_t)target & 0xff);                   // Addi.
1412   return LI64_INSTRUCTIONS_NUM * NativeInstruction::instruction_size;
1413 }
1414 
1415 static int patch_imm_in_li16u(address branch, uint16_t target) {
1416   Assembler::patch(branch, 31, 12, target); // patch lui only
1417   return NativeInstruction::instruction_size;
1418 }
1419 
1420 int MacroAssembler::patch_imm_in_li32(address branch, int32_t target) {
1421   const int LI32_INSTRUCTIONS_NUM = 2;                                          // lui + addiw
1422   int64_t upper = (intptr_t)target;
1423   int32_t lower = (((int32_t)target) << 20) >> 20;
1424   upper -= lower;
1425   upper = (int32_t)upper;
1426   Assembler::patch(branch + 0,  31, 12, (upper >> 12) & 0xfffff);               // Lui.
1427   Assembler::patch(branch + 4,  31, 20, lower & 0xfff);                         // Addiw.
1428   return LI32_INSTRUCTIONS_NUM * NativeInstruction::instruction_size;
1429 }
1430 
1431 static long get_offset_of_jal(address insn_addr) {
1432   assert_cond(insn_addr != nullptr);
1433   long offset = 0;
1434   unsigned insn = Assembler::ld_instr(insn_addr);
1435   long val = (long)Assembler::sextract(insn, 31, 12);
1436   offset |= ((val >> 19) & 0x1) << 20;
1437   offset |= (val & 0xff) << 12;
1438   offset |= ((val >> 8) & 0x1) << 11;
1439   offset |= ((val >> 9) & 0x3ff) << 1;
1440   offset = (offset << 43) >> 43;
1441   return offset;
1442 }
1443 
1444 static long get_offset_of_conditional_branch(address insn_addr) {
1445   long offset = 0;
1446   assert_cond(insn_addr != nullptr);
1447   unsigned insn = Assembler::ld_instr(insn_addr);
1448   offset = (long)Assembler::sextract(insn, 31, 31);
1449   offset = (offset << 12) | (((long)(Assembler::sextract(insn, 7, 7) & 0x1)) << 11);
1450   offset = offset | (((long)(Assembler::sextract(insn, 30, 25) & 0x3f)) << 5);
1451   offset = offset | (((long)(Assembler::sextract(insn, 11, 8) & 0xf)) << 1);
1452   offset = (offset << 41) >> 41;
1453   return offset;
1454 }
1455 
1456 static long get_offset_of_pc_relative(address insn_addr) {
1457   long offset = 0;
1458   assert_cond(insn_addr != nullptr);
1459   offset = ((long)(Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12))) << 12;                               // Auipc.
1460   offset += ((long)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20));                                  // Addi/Jalr/Load.
1461   offset = (offset << 32) >> 32;
1462   return offset;
1463 }
1464 
1465 static address get_target_of_movptr(address insn_addr) {
1466   assert_cond(insn_addr != nullptr);
1467   intptr_t target_address = (((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12)) & 0xfffff) << 29; // Lui.
1468   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20)) << 17;                 // Addi.
1469   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 12), 31, 20)) << 6;                 // Addi.
1470   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 20), 31, 20));                      // Addi/Jalr/Load.
1471   return (address) target_address;
1472 }
1473 
1474 static address get_target_of_li64(address insn_addr) {
1475   assert_cond(insn_addr != nullptr);
1476   intptr_t target_address = (((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12)) & 0xfffff) << 44; // Lui.
1477   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20)) << 32;                 // Addi.
1478   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 12), 31, 20)) << 20;                // Addi.
1479   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 20), 31, 20)) << 8;                 // Addi.
1480   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 28), 31, 20));                      // Addi.
1481   return (address)target_address;
1482 }
1483 
1484 address MacroAssembler::get_target_of_li32(address insn_addr) {
1485   assert_cond(insn_addr != nullptr);
1486   intptr_t target_address = (((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr), 31, 12)) & 0xfffff) << 12; // Lui.
1487   target_address += ((int64_t)Assembler::sextract(Assembler::ld_instr(insn_addr + 4), 31, 20));                       // Addiw.
1488   return (address)target_address;
1489 }
1490 
1491 // Patch any kind of instruction; there may be several instructions.
1492 // Return the total length (in bytes) of the instructions.
1493 int MacroAssembler::pd_patch_instruction_size(address branch, address target) {
1494   assert_cond(branch != nullptr);
1495   int64_t offset = target - branch;
1496   if (NativeInstruction::is_jal_at(branch)) {                         // jal
1497     return patch_offset_in_jal(branch, offset);
1498   } else if (NativeInstruction::is_branch_at(branch)) {               // beq/bge/bgeu/blt/bltu/bne
1499     return patch_offset_in_conditional_branch(branch, offset);
1500   } else if (NativeInstruction::is_pc_relative_at(branch)) {          // auipc, addi/jalr/load
1501     return patch_offset_in_pc_relative(branch, offset);
1502   } else if (NativeInstruction::is_movptr_at(branch)) {               // movptr
1503     return patch_addr_in_movptr(branch, target);
1504   } else if (NativeInstruction::is_li64_at(branch)) {                 // li64
1505     return patch_imm_in_li64(branch, target);
1506   } else if (NativeInstruction::is_li32_at(branch)) {                 // li32
1507     int64_t imm = (intptr_t)target;
1508     return patch_imm_in_li32(branch, (int32_t)imm);
1509   } else if (NativeInstruction::is_li16u_at(branch)) {
1510     int64_t imm = (intptr_t)target;
1511     return patch_imm_in_li16u(branch, (uint16_t)imm);
1512   } else {
1513 #ifdef ASSERT
1514     tty->print_cr("pd_patch_instruction_size: instruction 0x%x at " INTPTR_FORMAT " could not be patched!\n",
1515                   Assembler::ld_instr(branch), p2i(branch));
1516     Disassembler::decode(branch - 16, branch + 16);
1517 #endif
1518     ShouldNotReachHere();
1519     return -1;
1520   }
1521 }
1522 
1523 address MacroAssembler::target_addr_for_insn(address insn_addr) {
1524   long offset = 0;
1525   assert_cond(insn_addr != nullptr);
1526   if (NativeInstruction::is_jal_at(insn_addr)) {                     // jal
1527     offset = get_offset_of_jal(insn_addr);
1528   } else if (NativeInstruction::is_branch_at(insn_addr)) {           // beq/bge/bgeu/blt/bltu/bne
1529     offset = get_offset_of_conditional_branch(insn_addr);
1530   } else if (NativeInstruction::is_pc_relative_at(insn_addr)) {      // auipc, addi/jalr/load
1531     offset = get_offset_of_pc_relative(insn_addr);
1532   } else if (NativeInstruction::is_movptr_at(insn_addr)) {           // movptr
1533     return get_target_of_movptr(insn_addr);
1534   } else if (NativeInstruction::is_li64_at(insn_addr)) {             // li64
1535     return get_target_of_li64(insn_addr);
1536   } else if (NativeInstruction::is_li32_at(insn_addr)) {             // li32
1537     return get_target_of_li32(insn_addr);
1538   } else {
1539     ShouldNotReachHere();
1540   }
1541   return address(((uintptr_t)insn_addr + offset));
1542 }
1543 
1544 int MacroAssembler::patch_oop(address insn_addr, address o) {
1545   // OOPs are either narrow (32 bits) or wide (48 bits).  We encode
1546   // narrow OOPs by setting the upper 16 bits in the first
1547   // instruction.
1548   if (NativeInstruction::is_li32_at(insn_addr)) {
1549     // Move narrow OOP
1550     uint32_t n = CompressedOops::narrow_oop_value(cast_to_oop(o));
1551     return patch_imm_in_li32(insn_addr, (int32_t)n);
1552   } else if (NativeInstruction::is_movptr_at(insn_addr)) {
1553     // Move wide OOP
1554     return patch_addr_in_movptr(insn_addr, o);
1555   }
1556   ShouldNotReachHere();
1557   return -1;
1558 }
1559 
1560 void MacroAssembler::reinit_heapbase() {
1561   if (UseCompressedOops) {
1562     if (Universe::is_fully_initialized()) {
1563       mv(xheapbase, CompressedOops::ptrs_base());
1564     } else {
1565       ExternalAddress target(CompressedOops::ptrs_base_addr());
1566       relocate(target.rspec(), [&] {
1567         int32_t offset;
1568         la_patchable(xheapbase, target, offset);
1569         ld(xheapbase, Address(xheapbase, offset));
1570       });
1571     }
1572   }
1573 }
1574 
1575 void MacroAssembler::movptr(Register Rd, address addr, int32_t &offset) {
1576   int64_t imm64 = (int64_t)addr;
1577 #ifndef PRODUCT
1578   {
1579     char buffer[64];
1580     snprintf(buffer, sizeof(buffer), "0x%" PRIx64, imm64);
1581     block_comment(buffer);
1582   }
1583 #endif
1584   assert((uintptr_t)imm64 < (1ull << 48), "48-bit overflow in address constant");
1585   // Load upper 31 bits
1586   int64_t imm = imm64 >> 17;
1587   int64_t upper = imm, lower = imm;
1588   lower = (lower << 52) >> 52;
1589   upper -= lower;
1590   upper = (int32_t)upper;
1591   lui(Rd, upper);
1592   addi(Rd, Rd, lower);
1593 
1594   // Load the rest 17 bits.
1595   slli(Rd, Rd, 11);
1596   addi(Rd, Rd, (imm64 >> 6) & 0x7ff);
1597   slli(Rd, Rd, 6);
1598 
1599   // This offset will be used by following jalr/ld.
1600   offset = imm64 & 0x3f;
1601 }
1602 
1603 void MacroAssembler::add(Register Rd, Register Rn, int64_t increment, Register temp) {
1604   if (is_simm12(increment)) {
1605     addi(Rd, Rn, increment);
1606   } else {
1607     assert_different_registers(Rn, temp);
1608     li(temp, increment);
1609     add(Rd, Rn, temp);
1610   }
1611 }
1612 
1613 void MacroAssembler::addw(Register Rd, Register Rn, int32_t increment, Register temp) {
1614   if (is_simm12(increment)) {
1615     addiw(Rd, Rn, increment);
1616   } else {
1617     assert_different_registers(Rn, temp);
1618     li(temp, increment);
1619     addw(Rd, Rn, temp);
1620   }
1621 }
1622 
1623 void MacroAssembler::sub(Register Rd, Register Rn, int64_t decrement, Register temp) {
1624   if (is_simm12(-decrement)) {
1625     addi(Rd, Rn, -decrement);
1626   } else {
1627     assert_different_registers(Rn, temp);
1628     li(temp, decrement);
1629     sub(Rd, Rn, temp);
1630   }
1631 }
1632 
1633 void MacroAssembler::subw(Register Rd, Register Rn, int32_t decrement, Register temp) {
1634   if (is_simm12(-decrement)) {
1635     addiw(Rd, Rn, -decrement);
1636   } else {
1637     assert_different_registers(Rn, temp);
1638     li(temp, decrement);
1639     subw(Rd, Rn, temp);
1640   }
1641 }
1642 
1643 void MacroAssembler::andrw(Register Rd, Register Rs1, Register Rs2) {
1644   andr(Rd, Rs1, Rs2);
1645   sign_extend(Rd, Rd, 32);
1646 }
1647 
1648 void MacroAssembler::orrw(Register Rd, Register Rs1, Register Rs2) {
1649   orr(Rd, Rs1, Rs2);
1650   sign_extend(Rd, Rd, 32);
1651 }
1652 
1653 void MacroAssembler::xorrw(Register Rd, Register Rs1, Register Rs2) {
1654   xorr(Rd, Rs1, Rs2);
1655   sign_extend(Rd, Rd, 32);
1656 }
1657 
1658 // Rd = Rs1 & (~Rd2)
1659 void MacroAssembler::andn(Register Rd, Register Rs1, Register Rs2) {
1660   if (UseZbb) {
1661     Assembler::andn(Rd, Rs1, Rs2);
1662     return;
1663   }
1664 
1665   notr(Rd, Rs2);
1666   andr(Rd, Rs1, Rd);
1667 }
1668 
1669 // Rd = Rs1 | (~Rd2)
1670 void MacroAssembler::orn(Register Rd, Register Rs1, Register Rs2) {
1671   if (UseZbb) {
1672     Assembler::orn(Rd, Rs1, Rs2);
1673     return;
1674   }
1675 
1676   notr(Rd, Rs2);
1677   orr(Rd, Rs1, Rd);
1678 }
1679 
1680 // Note: load_unsigned_short used to be called load_unsigned_word.
1681 int MacroAssembler::load_unsigned_short(Register dst, Address src) {
1682   int off = offset();
1683   lhu(dst, src);
1684   return off;
1685 }
1686 
1687 int MacroAssembler::load_unsigned_byte(Register dst, Address src) {
1688   int off = offset();
1689   lbu(dst, src);
1690   return off;
1691 }
1692 
1693 int MacroAssembler::load_signed_short(Register dst, Address src) {
1694   int off = offset();
1695   lh(dst, src);
1696   return off;
1697 }
1698 
1699 int MacroAssembler::load_signed_byte(Register dst, Address src) {
1700   int off = offset();
1701   lb(dst, src);
1702   return off;
1703 }
1704 
1705 void MacroAssembler::load_sized_value(Register dst, Address src, size_t size_in_bytes, bool is_signed) {
1706   switch (size_in_bytes) {
1707     case  8:  ld(dst, src); break;
1708     case  4:  is_signed ? lw(dst, src) : lwu(dst, src); break;
1709     case  2:  is_signed ? load_signed_short(dst, src) : load_unsigned_short(dst, src); break;
1710     case  1:  is_signed ? load_signed_byte( dst, src) : load_unsigned_byte( dst, src); break;
1711     default:  ShouldNotReachHere();
1712   }
1713 }
1714 
1715 void MacroAssembler::store_sized_value(Address dst, Register src, size_t size_in_bytes) {
1716   switch (size_in_bytes) {
1717     case  8:  sd(src, dst); break;
1718     case  4:  sw(src, dst); break;
1719     case  2:  sh(src, dst); break;
1720     case  1:  sb(src, dst); break;
1721     default:  ShouldNotReachHere();
1722   }
1723 }
1724 
1725 // granularity is 1 OR 2 bytes per load. dst and src.base() allowed to be the same register
1726 void MacroAssembler::load_short_misaligned(Register dst, Address src, Register tmp, bool is_signed, int granularity) {
1727   if (granularity != 1 && granularity != 2) {
1728     ShouldNotReachHere();
1729   }
1730   if (AvoidUnalignedAccesses && (granularity != 2)) {
1731     assert_different_registers(dst, tmp);
1732     assert_different_registers(tmp, src.base());
1733     is_signed ? lb(tmp, Address(src.base(), src.offset() + 1)) : lbu(tmp, Address(src.base(), src.offset() + 1));
1734     slli(tmp, tmp, 8);
1735     lbu(dst, src);
1736     add(dst, dst, tmp);
1737   } else {
1738     is_signed ? lh(dst, src) : lhu(dst, src);
1739   }
1740 }
1741 
1742 // granularity is 1, 2 OR 4 bytes per load, if granularity 2 or 4 then dst and src.base() allowed to be the same register
1743 void MacroAssembler::load_int_misaligned(Register dst, Address src, Register tmp, bool is_signed, int granularity) {
1744   if (AvoidUnalignedAccesses && (granularity != 4)) {
1745     switch(granularity) {
1746       case 1:
1747         assert_different_registers(dst, tmp, src.base());
1748         lbu(dst, src);
1749         lbu(tmp, Address(src.base(), src.offset() + 1));
1750         slli(tmp, tmp, 8);
1751         add(dst, dst, tmp);
1752         lbu(tmp, Address(src.base(), src.offset() + 2));
1753         slli(tmp, tmp, 16);
1754         add(dst, dst, tmp);
1755         is_signed ? lb(tmp, Address(src.base(), src.offset() + 3)) : lbu(tmp, Address(src.base(), src.offset() + 3));
1756         slli(tmp, tmp, 24);
1757         add(dst, dst, tmp);
1758         break;
1759       case 2:
1760         assert_different_registers(dst, tmp);
1761         assert_different_registers(tmp, src.base());
1762         is_signed ? lh(tmp, Address(src.base(), src.offset() + 2)) : lhu(tmp, Address(src.base(), src.offset() + 2));
1763         slli(tmp, tmp, 16);
1764         lhu(dst, src);
1765         add(dst, dst, tmp);
1766         break;
1767       default:
1768         ShouldNotReachHere();
1769     }
1770   } else {
1771     is_signed ? lw(dst, src) : lwu(dst, src);
1772   }
1773 }
1774 
1775 // granularity is 1, 2, 4 or 8 bytes per load, if granularity 4 or 8 then dst and src.base() allowed to be same register
1776 void MacroAssembler::load_long_misaligned(Register dst, Address src, Register tmp, int granularity) {
1777   if (AvoidUnalignedAccesses && (granularity != 8)) {
1778     switch(granularity){
1779       case 1:
1780         assert_different_registers(dst, tmp, src.base());
1781         lbu(dst, src);
1782         lbu(tmp, Address(src.base(), src.offset() + 1));
1783         slli(tmp, tmp, 8);
1784         add(dst, dst, tmp);
1785         lbu(tmp, Address(src.base(), src.offset() + 2));
1786         slli(tmp, tmp, 16);
1787         add(dst, dst, tmp);
1788         lbu(tmp, Address(src.base(), src.offset() + 3));
1789         slli(tmp, tmp, 24);
1790         add(dst, dst, tmp);
1791         lbu(tmp, Address(src.base(), src.offset() + 4));
1792         slli(tmp, tmp, 32);
1793         add(dst, dst, tmp);
1794         lbu(tmp, Address(src.base(), src.offset() + 5));
1795         slli(tmp, tmp, 40);
1796         add(dst, dst, tmp);
1797         lbu(tmp, Address(src.base(), src.offset() + 6));
1798         slli(tmp, tmp, 48);
1799         add(dst, dst, tmp);
1800         lbu(tmp, Address(src.base(), src.offset() + 7));
1801         slli(tmp, tmp, 56);
1802         add(dst, dst, tmp);
1803         break;
1804       case 2:
1805         assert_different_registers(dst, tmp, src.base());
1806         lhu(dst, src);
1807         lhu(tmp, Address(src.base(), src.offset() + 2));
1808         slli(tmp, tmp, 16);
1809         add(dst, dst, tmp);
1810         lhu(tmp, Address(src.base(), src.offset() + 4));
1811         slli(tmp, tmp, 32);
1812         add(dst, dst, tmp);
1813         lhu(tmp, Address(src.base(), src.offset() + 6));
1814         slli(tmp, tmp, 48);
1815         add(dst, dst, tmp);
1816         break;
1817       case 4:
1818         assert_different_registers(dst, tmp);
1819         assert_different_registers(tmp, src.base());
1820         lwu(tmp, Address(src.base(), src.offset() + 4));
1821         slli(tmp, tmp, 32);
1822         lwu(dst, src);
1823         add(dst, dst, tmp);
1824         break;
1825       default:
1826         ShouldNotReachHere();
1827     }
1828   } else {
1829     ld(dst, src);
1830   }
1831 }
1832 
1833 
1834 // reverse bytes in halfword in lower 16 bits and sign-extend
1835 // Rd[15:0] = Rs[7:0] Rs[15:8] (sign-extend to 64 bits)
1836 void MacroAssembler::revb_h_h(Register Rd, Register Rs, Register tmp) {
1837   if (UseZbb) {
1838     rev8(Rd, Rs);
1839     srai(Rd, Rd, 48);
1840     return;
1841   }
1842   assert_different_registers(Rs, tmp);
1843   assert_different_registers(Rd, tmp);
1844   srli(tmp, Rs, 8);
1845   andi(tmp, tmp, 0xFF);
1846   slli(Rd, Rs, 56);
1847   srai(Rd, Rd, 48); // sign-extend
1848   orr(Rd, Rd, tmp);
1849 }
1850 
1851 // reverse bytes in lower word and sign-extend
1852 // Rd[31:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] (sign-extend to 64 bits)
1853 void MacroAssembler::revb_w_w(Register Rd, Register Rs, Register tmp1, Register tmp2) {
1854   if (UseZbb) {
1855     rev8(Rd, Rs);
1856     srai(Rd, Rd, 32);
1857     return;
1858   }
1859   assert_different_registers(Rs, tmp1, tmp2);
1860   assert_different_registers(Rd, tmp1, tmp2);
1861   revb_h_w_u(Rd, Rs, tmp1, tmp2);
1862   slli(tmp2, Rd, 48);
1863   srai(tmp2, tmp2, 32); // sign-extend
1864   srli(Rd, Rd, 16);
1865   orr(Rd, Rd, tmp2);
1866 }
1867 
1868 // reverse bytes in halfword in lower 16 bits and zero-extend
1869 // Rd[15:0] = Rs[7:0] Rs[15:8] (zero-extend to 64 bits)
1870 void MacroAssembler::revb_h_h_u(Register Rd, Register Rs, Register tmp) {
1871   if (UseZbb) {
1872     rev8(Rd, Rs);
1873     srli(Rd, Rd, 48);
1874     return;
1875   }
1876   assert_different_registers(Rs, tmp);
1877   assert_different_registers(Rd, tmp);
1878   srli(tmp, Rs, 8);
1879   andi(tmp, tmp, 0xFF);
1880   andi(Rd, Rs, 0xFF);
1881   slli(Rd, Rd, 8);
1882   orr(Rd, Rd, tmp);
1883 }
1884 
1885 // reverse bytes in halfwords in lower 32 bits and zero-extend
1886 // Rd[31:0] = Rs[23:16] Rs[31:24] Rs[7:0] Rs[15:8] (zero-extend to 64 bits)
1887 void MacroAssembler::revb_h_w_u(Register Rd, Register Rs, Register tmp1, Register tmp2) {
1888   if (UseZbb) {
1889     rev8(Rd, Rs);
1890     rori(Rd, Rd, 32);
1891     roriw(Rd, Rd, 16);
1892     zero_extend(Rd, Rd, 32);
1893     return;
1894   }
1895   assert_different_registers(Rs, tmp1, tmp2);
1896   assert_different_registers(Rd, tmp1, tmp2);
1897   srli(tmp2, Rs, 16);
1898   revb_h_h_u(tmp2, tmp2, tmp1);
1899   revb_h_h_u(Rd, Rs, tmp1);
1900   slli(tmp2, tmp2, 16);
1901   orr(Rd, Rd, tmp2);
1902 }
1903 
1904 // This method is only used for revb_h
1905 // Rd = Rs[47:0] Rs[55:48] Rs[63:56]
1906 void MacroAssembler::revb_h_helper(Register Rd, Register Rs, Register tmp1, Register tmp2) {
1907   assert_different_registers(Rs, tmp1, tmp2);
1908   assert_different_registers(Rd, tmp1);
1909   srli(tmp1, Rs, 48);
1910   andi(tmp2, tmp1, 0xFF);
1911   slli(tmp2, tmp2, 8);
1912   srli(tmp1, tmp1, 8);
1913   orr(tmp1, tmp1, tmp2);
1914   slli(Rd, Rs, 16);
1915   orr(Rd, Rd, tmp1);
1916 }
1917 
1918 // reverse bytes in each halfword
1919 // Rd[63:0] = Rs[55:48] Rs[63:56] Rs[39:32] Rs[47:40] Rs[23:16] Rs[31:24] Rs[7:0] Rs[15:8]
1920 void MacroAssembler::revb_h(Register Rd, Register Rs, Register tmp1, Register tmp2) {
1921   if (UseZbb) {
1922     assert_different_registers(Rs, tmp1);
1923     assert_different_registers(Rd, tmp1);
1924     rev8(Rd, Rs);
1925     zero_extend(tmp1, Rd, 32);
1926     roriw(tmp1, tmp1, 16);
1927     slli(tmp1, tmp1, 32);
1928     srli(Rd, Rd, 32);
1929     roriw(Rd, Rd, 16);
1930     zero_extend(Rd, Rd, 32);
1931     orr(Rd, Rd, tmp1);
1932     return;
1933   }
1934   assert_different_registers(Rs, tmp1, tmp2);
1935   assert_different_registers(Rd, tmp1, tmp2);
1936   revb_h_helper(Rd, Rs, tmp1, tmp2);
1937   for (int i = 0; i < 3; ++i) {
1938     revb_h_helper(Rd, Rd, tmp1, tmp2);
1939   }
1940 }
1941 
1942 // reverse bytes in each word
1943 // Rd[63:0] = Rs[39:32] Rs[47:40] Rs[55:48] Rs[63:56] Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24]
1944 void MacroAssembler::revb_w(Register Rd, Register Rs, Register tmp1, Register tmp2) {
1945   if (UseZbb) {
1946     rev8(Rd, Rs);
1947     rori(Rd, Rd, 32);
1948     return;
1949   }
1950   assert_different_registers(Rs, tmp1, tmp2);
1951   assert_different_registers(Rd, tmp1, tmp2);
1952   revb(Rd, Rs, tmp1, tmp2);
1953   ror_imm(Rd, Rd, 32);
1954 }
1955 
1956 // reverse bytes in doubleword
1957 // Rd[63:0] = Rs[7:0] Rs[15:8] Rs[23:16] Rs[31:24] Rs[39:32] Rs[47,40] Rs[55,48] Rs[63:56]
1958 void MacroAssembler::revb(Register Rd, Register Rs, Register tmp1, Register tmp2) {
1959   if (UseZbb) {
1960     rev8(Rd, Rs);
1961     return;
1962   }
1963   assert_different_registers(Rs, tmp1, tmp2);
1964   assert_different_registers(Rd, tmp1, tmp2);
1965   andi(tmp1, Rs, 0xFF);
1966   slli(tmp1, tmp1, 8);
1967   for (int step = 8; step < 56; step += 8) {
1968     srli(tmp2, Rs, step);
1969     andi(tmp2, tmp2, 0xFF);
1970     orr(tmp1, tmp1, tmp2);
1971     slli(tmp1, tmp1, 8);
1972   }
1973   srli(Rd, Rs, 56);
1974   andi(Rd, Rd, 0xFF);
1975   orr(Rd, tmp1, Rd);
1976 }
1977 
1978 // rotate right with shift bits
1979 void MacroAssembler::ror_imm(Register dst, Register src, uint32_t shift, Register tmp)
1980 {
1981   if (UseZbb) {
1982     rori(dst, src, shift);
1983     return;
1984   }
1985 
1986   assert_different_registers(dst, tmp);
1987   assert_different_registers(src, tmp);
1988   assert(shift < 64, "shift amount must be < 64");
1989   slli(tmp, src, 64 - shift);
1990   srli(dst, src, shift);
1991   orr(dst, dst, tmp);
1992 }
1993 
1994 // rotate left with shift bits, 32-bit version
1995 void MacroAssembler::rolw_imm(Register dst, Register src, uint32_t shift, Register tmp) {
1996   if (UseZbb) {
1997     // no roliw available
1998     roriw(dst, src, 32 - shift);
1999     return;
2000   }
2001 
2002   assert_different_registers(dst, tmp);
2003   assert_different_registers(src, tmp);
2004   assert(shift < 32, "shift amount must be < 32");
2005   srliw(tmp, src, 32 - shift);
2006   slliw(dst, src, shift);
2007   orr(dst, dst, tmp);
2008 }
2009 
2010 void MacroAssembler::andi(Register Rd, Register Rn, int64_t imm, Register tmp) {
2011   if (is_simm12(imm)) {
2012     and_imm12(Rd, Rn, imm);
2013   } else {
2014     assert_different_registers(Rn, tmp);
2015     mv(tmp, imm);
2016     andr(Rd, Rn, tmp);
2017   }
2018 }
2019 
2020 void MacroAssembler::orptr(Address adr, RegisterOrConstant src, Register tmp1, Register tmp2) {
2021   ld(tmp1, adr);
2022   if (src.is_register()) {
2023     orr(tmp1, tmp1, src.as_register());
2024   } else {
2025     if (is_simm12(src.as_constant())) {
2026       ori(tmp1, tmp1, src.as_constant());
2027     } else {
2028       assert_different_registers(tmp1, tmp2);
2029       mv(tmp2, src.as_constant());
2030       orr(tmp1, tmp1, tmp2);
2031     }
2032   }
2033   sd(tmp1, adr);
2034 }
2035 
2036 void MacroAssembler::cmp_klass(Register oop, Register trial_klass, Register tmp1, Register tmp2, Label &L) {
2037   assert_different_registers(oop, trial_klass, tmp1, tmp2);
2038   if (UseCompressedClassPointers) {
2039     lwu(tmp1, Address(oop, oopDesc::klass_offset_in_bytes()));
2040     if (CompressedKlassPointers::base() == nullptr) {
2041       slli(tmp1, tmp1, CompressedKlassPointers::shift());
2042       beq(trial_klass, tmp1, L);
2043       return;
2044     }
2045     decode_klass_not_null(tmp1, tmp2);
2046   } else {
2047     ld(tmp1, Address(oop, oopDesc::klass_offset_in_bytes()));
2048   }
2049   beq(trial_klass, tmp1, L);
2050 }
2051 
2052 // Move an oop into a register.
2053 void MacroAssembler::movoop(Register dst, jobject obj) {
2054   int oop_index;
2055   if (obj == nullptr) {
2056     oop_index = oop_recorder()->allocate_oop_index(obj);
2057   } else {
2058 #ifdef ASSERT
2059     {
2060       ThreadInVMfromUnknown tiv;
2061       assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop");
2062     }
2063 #endif
2064     oop_index = oop_recorder()->find_index(obj);
2065   }
2066   RelocationHolder rspec = oop_Relocation::spec(oop_index);
2067 
2068   if (BarrierSet::barrier_set()->barrier_set_assembler()->supports_instruction_patching()) {
2069     mv(dst, Address((address)obj, rspec));
2070   } else {
2071     address dummy = address(uintptr_t(pc()) & -wordSize); // A nearby aligned address
2072     ld_constant(dst, Address(dummy, rspec));
2073   }
2074 }
2075 
2076 // Move a metadata address into a register.
2077 void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
2078   int oop_index;
2079   if (obj == nullptr) {
2080     oop_index = oop_recorder()->allocate_metadata_index(obj);
2081   } else {
2082     oop_index = oop_recorder()->find_index(obj);
2083   }
2084   RelocationHolder rspec = metadata_Relocation::spec(oop_index);
2085   mv(dst, Address((address)obj, rspec));
2086 }
2087 
2088 // Writes to stack successive pages until offset reached to check for
2089 // stack overflow + shadow pages.  This clobbers tmp.
2090 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
2091   assert_different_registers(tmp, size, t0);
2092   // Bang stack for total size given plus shadow page size.
2093   // Bang one page at a time because large size can bang beyond yellow and
2094   // red zones.
2095   mv(t0, (int)os::vm_page_size());
2096   Label loop;
2097   bind(loop);
2098   sub(tmp, sp, t0);
2099   subw(size, size, t0);
2100   sd(size, Address(tmp));
2101   bgtz(size, loop);
2102 
2103   // Bang down shadow pages too.
2104   // At this point, (tmp-0) is the last address touched, so don't
2105   // touch it again.  (It was touched as (tmp-pagesize) but then tmp
2106   // was post-decremented.)  Skip this address by starting at i=1, and
2107   // touch a few more pages below.  N.B.  It is important to touch all
2108   // the way down to and including i=StackShadowPages.
2109   for (int i = 0; i < (int)(StackOverflow::stack_shadow_zone_size() / (int)os::vm_page_size()) - 1; i++) {
2110     // this could be any sized move but this is can be a debugging crumb
2111     // so the bigger the better.
2112     sub(tmp, tmp, (int)os::vm_page_size());
2113     sd(size, Address(tmp, 0));
2114   }
2115 }
2116 
2117 SkipIfEqual::SkipIfEqual(MacroAssembler* masm, const bool* flag_addr, bool value) {
2118   int32_t offset = 0;
2119   _masm = masm;
2120   ExternalAddress target((address)flag_addr);
2121   _masm->relocate(target.rspec(), [&] {
2122     int32_t offset;
2123     _masm->la_patchable(t0, target, offset);
2124     _masm->lbu(t0, Address(t0, offset));
2125   });
2126   if (value) {
2127     _masm->bnez(t0, _label);
2128   } else {
2129     _masm->beqz(t0, _label);
2130   }
2131 }
2132 
2133 SkipIfEqual::~SkipIfEqual() {
2134   _masm->bind(_label);
2135   _masm = nullptr;
2136 }
2137 
2138 void MacroAssembler::load_mirror(Register dst, Register method, Register tmp1, Register tmp2) {
2139   const int mirror_offset = in_bytes(Klass::java_mirror_offset());
2140   ld(dst, Address(xmethod, Method::const_offset()));
2141   ld(dst, Address(dst, ConstMethod::constants_offset()));
2142   ld(dst, Address(dst, ConstantPool::pool_holder_offset()));
2143   ld(dst, Address(dst, mirror_offset));
2144   resolve_oop_handle(dst, tmp1, tmp2);
2145 }
2146 
2147 void MacroAssembler::resolve_oop_handle(Register result, Register tmp1, Register tmp2) {
2148   // OopHandle::resolve is an indirection.
2149   assert_different_registers(result, tmp1, tmp2);
2150   access_load_at(T_OBJECT, IN_NATIVE, result, Address(result, 0), tmp1, tmp2);
2151 }
2152 
2153 // ((WeakHandle)result).resolve()
2154 void MacroAssembler::resolve_weak_handle(Register result, Register tmp1, Register tmp2) {
2155   assert_different_registers(result, tmp1, tmp2);
2156   Label resolved;
2157 
2158   // A null weak handle resolves to null.
2159   beqz(result, resolved);
2160 
2161   // Only 64 bit platforms support GCs that require a tmp register
2162   // Only IN_HEAP loads require a thread_tmp register
2163   // WeakHandle::resolve is an indirection like jweak.
2164   access_load_at(T_OBJECT, IN_NATIVE | ON_PHANTOM_OOP_REF,
2165                  result, Address(result), tmp1, tmp2);
2166   bind(resolved);
2167 }
2168 
2169 void MacroAssembler::access_load_at(BasicType type, DecoratorSet decorators,
2170                                     Register dst, Address src,
2171                                     Register tmp1, Register tmp2) {
2172   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2173   decorators = AccessInternal::decorator_fixup(decorators, type);
2174   bool as_raw = (decorators & AS_RAW) != 0;
2175   if (as_raw) {
2176     bs->BarrierSetAssembler::load_at(this, decorators, type, dst, src, tmp1, tmp2);
2177   } else {
2178     bs->load_at(this, decorators, type, dst, src, tmp1, tmp2);
2179   }
2180 }
2181 
2182 void MacroAssembler::null_check(Register reg, int offset) {
2183   if (needs_explicit_null_check(offset)) {
2184     // provoke OS null exception if reg is null by
2185     // accessing M[reg] w/o changing any registers
2186     // NOTE: this is plenty to provoke a segv
2187     ld(zr, Address(reg, 0));
2188   } else {
2189     // nothing to do, (later) access of M[reg + offset]
2190     // will provoke OS null exception if reg is null
2191   }
2192 }
2193 
2194 void MacroAssembler::access_store_at(BasicType type, DecoratorSet decorators,
2195                                      Address dst, Register val,
2196                                      Register tmp1, Register tmp2, Register tmp3) {
2197   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
2198   decorators = AccessInternal::decorator_fixup(decorators, type);
2199   bool as_raw = (decorators & AS_RAW) != 0;
2200   if (as_raw) {
2201     bs->BarrierSetAssembler::store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
2202   } else {
2203     bs->store_at(this, decorators, type, dst, val, tmp1, tmp2, tmp3);
2204   }
2205 }
2206 
2207 // Algorithm must match CompressedOops::encode.
2208 void MacroAssembler::encode_heap_oop(Register d, Register s) {
2209   verify_oop_msg(s, "broken oop in encode_heap_oop");
2210   if (CompressedOops::base() == nullptr) {
2211     if (CompressedOops::shift() != 0) {
2212       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
2213       srli(d, s, LogMinObjAlignmentInBytes);
2214     } else {
2215       mv(d, s);
2216     }
2217   } else {
2218     Label notNull;
2219     sub(d, s, xheapbase);
2220     bgez(d, notNull);
2221     mv(d, zr);
2222     bind(notNull);
2223     if (CompressedOops::shift() != 0) {
2224       assert (LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
2225       srli(d, d, CompressedOops::shift());
2226     }
2227   }
2228 }
2229 
2230 void MacroAssembler::load_klass(Register dst, Register src, Register tmp) {
2231   assert_different_registers(dst, tmp);
2232   assert_different_registers(src, tmp);
2233   if (UseCompressedClassPointers) {
2234     lwu(dst, Address(src, oopDesc::klass_offset_in_bytes()));
2235     decode_klass_not_null(dst, tmp);
2236   } else {
2237     ld(dst, Address(src, oopDesc::klass_offset_in_bytes()));
2238   }
2239 }
2240 
2241 void MacroAssembler::store_klass(Register dst, Register src, Register tmp) {
2242   // FIXME: Should this be a store release? concurrent gcs assumes
2243   // klass length is valid if klass field is not null.
2244   if (UseCompressedClassPointers) {
2245     encode_klass_not_null(src, tmp);
2246     sw(src, Address(dst, oopDesc::klass_offset_in_bytes()));
2247   } else {
2248     sd(src, Address(dst, oopDesc::klass_offset_in_bytes()));
2249   }
2250 }
2251 
2252 void MacroAssembler::store_klass_gap(Register dst, Register src) {
2253   if (UseCompressedClassPointers) {
2254     // Store to klass gap in destination
2255     sw(src, Address(dst, oopDesc::klass_gap_offset_in_bytes()));
2256   }
2257 }
2258 
2259 void MacroAssembler::decode_klass_not_null(Register r, Register tmp) {
2260   assert_different_registers(r, tmp);
2261   decode_klass_not_null(r, r, tmp);
2262 }
2263 
2264 void MacroAssembler::decode_klass_not_null(Register dst, Register src, Register tmp) {
2265   assert(UseCompressedClassPointers, "should only be used for compressed headers");
2266 
2267   if (CompressedKlassPointers::base() == nullptr) {
2268     if (CompressedKlassPointers::shift() != 0) {
2269       assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
2270       slli(dst, src, LogKlassAlignmentInBytes);
2271     } else {
2272       mv(dst, src);
2273     }
2274     return;
2275   }
2276 
2277   Register xbase = dst;
2278   if (dst == src) {
2279     xbase = tmp;
2280   }
2281 
2282   assert_different_registers(src, xbase);
2283   mv(xbase, (uintptr_t)CompressedKlassPointers::base());
2284 
2285   if (CompressedKlassPointers::shift() != 0) {
2286     assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
2287     assert_different_registers(t0, xbase);
2288     shadd(dst, src, xbase, t0, LogKlassAlignmentInBytes);
2289   } else {
2290     add(dst, xbase, src);
2291   }
2292 }
2293 
2294 void MacroAssembler::encode_klass_not_null(Register r, Register tmp) {
2295   assert_different_registers(r, tmp);
2296   encode_klass_not_null(r, r, tmp);
2297 }
2298 
2299 void MacroAssembler::encode_klass_not_null(Register dst, Register src, Register tmp) {
2300   assert(UseCompressedClassPointers, "should only be used for compressed headers");
2301 
2302   if (CompressedKlassPointers::base() == nullptr) {
2303     if (CompressedKlassPointers::shift() != 0) {
2304       assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
2305       srli(dst, src, LogKlassAlignmentInBytes);
2306     } else {
2307       mv(dst, src);
2308     }
2309     return;
2310   }
2311 
2312   if (((uint64_t)CompressedKlassPointers::base() & 0xffffffff) == 0 &&
2313       CompressedKlassPointers::shift() == 0) {
2314     zero_extend(dst, src, 32);
2315     return;
2316   }
2317 
2318   Register xbase = dst;
2319   if (dst == src) {
2320     xbase = tmp;
2321   }
2322 
2323   assert_different_registers(src, xbase);
2324   mv(xbase, (uintptr_t)CompressedKlassPointers::base());
2325   sub(dst, src, xbase);
2326   if (CompressedKlassPointers::shift() != 0) {
2327     assert(LogKlassAlignmentInBytes == CompressedKlassPointers::shift(), "decode alg wrong");
2328     srli(dst, dst, LogKlassAlignmentInBytes);
2329   }
2330 }
2331 
2332 void MacroAssembler::decode_heap_oop_not_null(Register r) {
2333   decode_heap_oop_not_null(r, r);
2334 }
2335 
2336 void MacroAssembler::decode_heap_oop_not_null(Register dst, Register src) {
2337   assert(UseCompressedOops, "should only be used for compressed headers");
2338   assert(Universe::heap() != nullptr, "java heap should be initialized");
2339   // Cannot assert, unverified entry point counts instructions (see .ad file)
2340   // vtableStubs also counts instructions in pd_code_size_limit.
2341   // Also do not verify_oop as this is called by verify_oop.
2342   if (CompressedOops::shift() != 0) {
2343     assert(LogMinObjAlignmentInBytes == CompressedOops::shift(), "decode alg wrong");
2344     slli(dst, src, LogMinObjAlignmentInBytes);
2345     if (CompressedOops::base() != nullptr) {
2346       add(dst, xheapbase, dst);
2347     }
2348   } else {
2349     assert(CompressedOops::base() == nullptr, "sanity");
2350     mv(dst, src);
2351   }
2352 }
2353 
2354 void  MacroAssembler::decode_heap_oop(Register d, Register s) {
2355   if (CompressedOops::base() == nullptr) {
2356     if (CompressedOops::shift() != 0 || d != s) {
2357       slli(d, s, CompressedOops::shift());
2358     }
2359   } else {
2360     Label done;
2361     mv(d, s);
2362     beqz(s, done);
2363     shadd(d, s, xheapbase, d, LogMinObjAlignmentInBytes);
2364     bind(done);
2365   }
2366   verify_oop_msg(d, "broken oop in decode_heap_oop");
2367 }
2368 
2369 void MacroAssembler::store_heap_oop(Address dst, Register val, Register tmp1,
2370                                     Register tmp2, Register tmp3, DecoratorSet decorators) {
2371   access_store_at(T_OBJECT, IN_HEAP | decorators, dst, val, tmp1, tmp2, tmp3);
2372 }
2373 
2374 void MacroAssembler::load_heap_oop(Register dst, Address src, Register tmp1,
2375                                    Register tmp2, DecoratorSet decorators) {
2376   access_load_at(T_OBJECT, IN_HEAP | decorators, dst, src, tmp1, tmp2);
2377 }
2378 
2379 void MacroAssembler::load_heap_oop_not_null(Register dst, Address src, Register tmp1,
2380                                             Register tmp2, DecoratorSet decorators) {
2381   access_load_at(T_OBJECT, IN_HEAP | IS_NOT_NULL, dst, src, tmp1, tmp2);
2382 }
2383 
2384 // Used for storing nulls.
2385 void MacroAssembler::store_heap_oop_null(Address dst) {
2386   access_store_at(T_OBJECT, IN_HEAP, dst, noreg, noreg, noreg, noreg);
2387 }
2388 
2389 int MacroAssembler::corrected_idivl(Register result, Register rs1, Register rs2,
2390                                     bool want_remainder)
2391 {
2392   // Full implementation of Java idiv and irem.  The function
2393   // returns the (pc) offset of the div instruction - may be needed
2394   // for implicit exceptions.
2395   //
2396   // input : rs1: dividend
2397   //         rs2: divisor
2398   //
2399   // result: either
2400   //         quotient  (= rs1 idiv rs2)
2401   //         remainder (= rs1 irem rs2)
2402 
2403 
2404   int idivl_offset = offset();
2405   if (!want_remainder) {
2406     divw(result, rs1, rs2);
2407   } else {
2408     remw(result, rs1, rs2); // result = rs1 % rs2;
2409   }
2410   return idivl_offset;
2411 }
2412 
2413 int MacroAssembler::corrected_idivq(Register result, Register rs1, Register rs2,
2414                                     bool want_remainder)
2415 {
2416   // Full implementation of Java ldiv and lrem.  The function
2417   // returns the (pc) offset of the div instruction - may be needed
2418   // for implicit exceptions.
2419   //
2420   // input : rs1: dividend
2421   //         rs2: divisor
2422   //
2423   // result: either
2424   //         quotient  (= rs1 idiv rs2)
2425   //         remainder (= rs1 irem rs2)
2426 
2427   int idivq_offset = offset();
2428   if (!want_remainder) {
2429     div(result, rs1, rs2);
2430   } else {
2431     rem(result, rs1, rs2); // result = rs1 % rs2;
2432   }
2433   return idivq_offset;
2434 }
2435 
2436 // Look up the method for a megamorpic invkkeinterface call.
2437 // The target method is determined by <intf_klass, itable_index>.
2438 // The receiver klass is in recv_klass.
2439 // On success, the result will be in method_result, and execution falls through.
2440 // On failure, execution transfers to the given label.
2441 void MacroAssembler::lookup_interface_method(Register recv_klass,
2442                                              Register intf_klass,
2443                                              RegisterOrConstant itable_index,
2444                                              Register method_result,
2445                                              Register scan_tmp,
2446                                              Label& L_no_such_interface,
2447                                              bool return_method) {
2448   assert_different_registers(recv_klass, intf_klass, scan_tmp);
2449   assert_different_registers(method_result, intf_klass, scan_tmp);
2450   assert(recv_klass != method_result || !return_method,
2451          "recv_klass can be destroyed when mehtid isn't needed");
2452   assert(itable_index.is_constant() || itable_index.as_register() == method_result,
2453          "caller must be same register for non-constant itable index as for method");
2454 
2455   // Compute start of first itableOffsetEntry (which is at the end of the vtable).
2456   int vtable_base = in_bytes(Klass::vtable_start_offset());
2457   int itentry_off = in_bytes(itableMethodEntry::method_offset());
2458   int scan_step   = itableOffsetEntry::size() * wordSize;
2459   int vte_size    = vtableEntry::size_in_bytes();
2460   assert(vte_size == wordSize, "else adjust times_vte_scale");
2461 
2462   lwu(scan_tmp, Address(recv_klass, Klass::vtable_length_offset()));
2463 
2464   // %%% Could store the aligned, prescaled offset in the klassoop.
2465   shadd(scan_tmp, scan_tmp, recv_klass, scan_tmp, 3);
2466   add(scan_tmp, scan_tmp, vtable_base);
2467 
2468   if (return_method) {
2469     // Adjust recv_klass by scaled itable_index, so we can free itable_index.
2470     assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
2471     if (itable_index.is_register()) {
2472       slli(t0, itable_index.as_register(), 3);
2473     } else {
2474       mv(t0, itable_index.as_constant() << 3);
2475     }
2476     add(recv_klass, recv_klass, t0);
2477     if (itentry_off) {
2478       add(recv_klass, recv_klass, itentry_off);
2479     }
2480   }
2481 
2482   Label search, found_method;
2483 
2484   ld(method_result, Address(scan_tmp, itableOffsetEntry::interface_offset()));
2485   beq(intf_klass, method_result, found_method);
2486   bind(search);
2487   // Check that the previous entry is non-null. A null entry means that
2488   // the receiver class doesn't implement the interface, and wasn't the
2489   // same as when the caller was compiled.
2490   beqz(method_result, L_no_such_interface, /* is_far */ true);
2491   addi(scan_tmp, scan_tmp, scan_step);
2492   ld(method_result, Address(scan_tmp, itableOffsetEntry::interface_offset()));
2493   bne(intf_klass, method_result, search);
2494 
2495   bind(found_method);
2496 
2497   // Got a hit.
2498   if (return_method) {
2499     lwu(scan_tmp, Address(scan_tmp, itableOffsetEntry::offset_offset()));
2500     add(method_result, recv_klass, scan_tmp);
2501     ld(method_result, Address(method_result));
2502   }
2503 }
2504 
2505 // virtual method calling
2506 void MacroAssembler::lookup_virtual_method(Register recv_klass,
2507                                            RegisterOrConstant vtable_index,
2508                                            Register method_result) {
2509   const ByteSize base = Klass::vtable_start_offset();
2510   assert(vtableEntry::size() * wordSize == 8,
2511          "adjust the scaling in the code below");
2512   int vtable_offset_in_bytes = in_bytes(base + vtableEntry::method_offset());
2513 
2514   if (vtable_index.is_register()) {
2515     shadd(method_result, vtable_index.as_register(), recv_klass, method_result, LogBytesPerWord);
2516     ld(method_result, Address(method_result, vtable_offset_in_bytes));
2517   } else {
2518     vtable_offset_in_bytes += vtable_index.as_constant() * wordSize;
2519     ld(method_result, form_address(method_result, recv_klass, vtable_offset_in_bytes));
2520   }
2521 }
2522 
2523 void MacroAssembler::membar(uint32_t order_constraint) {
2524   address prev = pc() - NativeMembar::instruction_size;
2525   address last = code()->last_insn();
2526 
2527   if (last != nullptr && nativeInstruction_at(last)->is_membar() && prev == last) {
2528     NativeMembar *bar = NativeMembar_at(prev);
2529     // We are merging two memory barrier instructions.  On RISCV we
2530     // can do this simply by ORing them together.
2531     bar->set_kind(bar->get_kind() | order_constraint);
2532     BLOCK_COMMENT("merged membar");
2533   } else {
2534     code()->set_last_insn(pc());
2535 
2536     uint32_t predecessor = 0;
2537     uint32_t successor = 0;
2538 
2539     membar_mask_to_pred_succ(order_constraint, predecessor, successor);
2540     fence(predecessor, successor);
2541   }
2542 }
2543 
2544 // Form an address from base + offset in Rd. Rd my or may not
2545 // actually be used: you must use the Address that is returned. It
2546 // is up to you to ensure that the shift provided matches the size
2547 // of your data.
2548 Address MacroAssembler::form_address(Register Rd, Register base, int64_t byte_offset) {
2549   if (is_simm12(byte_offset)) { // 12: imm in range 2^12
2550     return Address(base, byte_offset);
2551   }
2552 
2553   assert_different_registers(Rd, base, noreg);
2554 
2555   // Do it the hard way
2556   mv(Rd, byte_offset);
2557   add(Rd, base, Rd);
2558   return Address(Rd);
2559 }
2560 
2561 void MacroAssembler::check_klass_subtype(Register sub_klass,
2562                                          Register super_klass,
2563                                          Register tmp_reg,
2564                                          Label& L_success) {
2565   Label L_failure;
2566   check_klass_subtype_fast_path(sub_klass, super_klass, tmp_reg, &L_success, &L_failure, nullptr);
2567   check_klass_subtype_slow_path(sub_klass, super_klass, tmp_reg, noreg, &L_success, nullptr);
2568   bind(L_failure);
2569 }
2570 
2571 void MacroAssembler::safepoint_poll(Label& slow_path, bool at_return, bool acquire, bool in_nmethod) {
2572   ld(t0, Address(xthread, JavaThread::polling_word_offset()));
2573   if (acquire) {
2574     membar(MacroAssembler::LoadLoad | MacroAssembler::LoadStore);
2575   }
2576   if (at_return) {
2577     bgtu(in_nmethod ? sp : fp, t0, slow_path, /* is_far */ true);
2578   } else {
2579     test_bit(t0, t0, exact_log2(SafepointMechanism::poll_bit()));
2580     bnez(t0, slow_path, true /* is_far */);
2581   }
2582 }
2583 
2584 void MacroAssembler::cmpxchgptr(Register oldv, Register newv, Register addr, Register tmp,
2585                                 Label &succeed, Label *fail) {
2586   assert_different_registers(addr, tmp);
2587   assert_different_registers(newv, tmp);
2588   assert_different_registers(oldv, tmp);
2589 
2590   // oldv holds comparison value
2591   // newv holds value to write in exchange
2592   // addr identifies memory word to compare against/update
2593   Label retry_load, nope;
2594   bind(retry_load);
2595   // Load reserved from the memory location
2596   lr_d(tmp, addr, Assembler::aqrl);
2597   // Fail and exit if it is not what we expect
2598   bne(tmp, oldv, nope);
2599   // If the store conditional succeeds, tmp will be zero
2600   sc_d(tmp, newv, addr, Assembler::rl);
2601   beqz(tmp, succeed);
2602   // Retry only when the store conditional failed
2603   j(retry_load);
2604 
2605   bind(nope);
2606   membar(AnyAny);
2607   mv(oldv, tmp);
2608   if (fail != nullptr) {
2609     j(*fail);
2610   }
2611 }
2612 
2613 void MacroAssembler::cmpxchg_obj_header(Register oldv, Register newv, Register obj, Register tmp,
2614                                         Label &succeed, Label *fail) {
2615   assert(oopDesc::mark_offset_in_bytes() == 0, "assumption");
2616   cmpxchgptr(oldv, newv, obj, tmp, succeed, fail);
2617 }
2618 
2619 void MacroAssembler::load_reserved(Register addr,
2620                                    enum operand_size size,
2621                                    Assembler::Aqrl acquire) {
2622   switch (size) {
2623     case int64:
2624       lr_d(t0, addr, acquire);
2625       break;
2626     case int32:
2627       lr_w(t0, addr, acquire);
2628       break;
2629     case uint32:
2630       lr_w(t0, addr, acquire);
2631       zero_extend(t0, t0, 32);
2632       break;
2633     default:
2634       ShouldNotReachHere();
2635   }
2636 }
2637 
2638 void MacroAssembler::store_conditional(Register addr,
2639                                        Register new_val,
2640                                        enum operand_size size,
2641                                        Assembler::Aqrl release) {
2642   switch (size) {
2643     case int64:
2644       sc_d(t0, new_val, addr, release);
2645       break;
2646     case int32:
2647     case uint32:
2648       sc_w(t0, new_val, addr, release);
2649       break;
2650     default:
2651       ShouldNotReachHere();
2652   }
2653 }
2654 
2655 
2656 void MacroAssembler::cmpxchg_narrow_value_helper(Register addr, Register expected,
2657                                                  Register new_val,
2658                                                  enum operand_size size,
2659                                                  Register tmp1, Register tmp2, Register tmp3) {
2660   assert(size == int8 || size == int16, "unsupported operand size");
2661 
2662   Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3;
2663 
2664   andi(shift, addr, 3);
2665   slli(shift, shift, 3);
2666 
2667   andi(aligned_addr, addr, ~3);
2668 
2669   if (size == int8) {
2670     mv(mask, 0xff);
2671   } else {
2672     // size == int16 case
2673     mv(mask, -1);
2674     zero_extend(mask, mask, 16);
2675   }
2676   sll(mask, mask, shift);
2677 
2678   xori(not_mask, mask, -1);
2679 
2680   sll(expected, expected, shift);
2681   andr(expected, expected, mask);
2682 
2683   sll(new_val, new_val, shift);
2684   andr(new_val, new_val, mask);
2685 }
2686 
2687 // cmpxchg_narrow_value will kill t0, t1, expected, new_val and tmps.
2688 // It's designed to implement compare and swap byte/boolean/char/short by lr.w/sc.w,
2689 // which are forced to work with 4-byte aligned address.
2690 void MacroAssembler::cmpxchg_narrow_value(Register addr, Register expected,
2691                                           Register new_val,
2692                                           enum operand_size size,
2693                                           Assembler::Aqrl acquire, Assembler::Aqrl release,
2694                                           Register result, bool result_as_bool,
2695                                           Register tmp1, Register tmp2, Register tmp3) {
2696   Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0;
2697   assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp);
2698   cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3);
2699 
2700   Label retry, fail, done;
2701 
2702   bind(retry);
2703   lr_w(old, aligned_addr, acquire);
2704   andr(tmp, old, mask);
2705   bne(tmp, expected, fail);
2706 
2707   andr(tmp, old, not_mask);
2708   orr(tmp, tmp, new_val);
2709   sc_w(tmp, tmp, aligned_addr, release);
2710   bnez(tmp, retry);
2711 
2712   if (result_as_bool) {
2713     mv(result, 1);
2714     j(done);
2715 
2716     bind(fail);
2717     mv(result, zr);
2718 
2719     bind(done);
2720   } else {
2721     andr(tmp, old, mask);
2722 
2723     bind(fail);
2724     srl(result, tmp, shift);
2725 
2726     if (size == int8) {
2727       sign_extend(result, result, 8);
2728     } else {
2729       // size == int16 case
2730       sign_extend(result, result, 16);
2731     }
2732   }
2733 }
2734 
2735 // weak_cmpxchg_narrow_value is a weak version of cmpxchg_narrow_value, to implement
2736 // the weak CAS stuff. The major difference is that it just failed when store conditional
2737 // failed.
2738 void MacroAssembler::weak_cmpxchg_narrow_value(Register addr, Register expected,
2739                                                Register new_val,
2740                                                enum operand_size size,
2741                                                Assembler::Aqrl acquire, Assembler::Aqrl release,
2742                                                Register result,
2743                                                Register tmp1, Register tmp2, Register tmp3) {
2744   Register aligned_addr = t1, shift = tmp1, mask = tmp2, not_mask = tmp3, old = result, tmp = t0;
2745   assert_different_registers(addr, old, mask, not_mask, new_val, expected, shift, tmp);
2746   cmpxchg_narrow_value_helper(addr, expected, new_val, size, tmp1, tmp2, tmp3);
2747 
2748   Label fail, done;
2749 
2750   lr_w(old, aligned_addr, acquire);
2751   andr(tmp, old, mask);
2752   bne(tmp, expected, fail);
2753 
2754   andr(tmp, old, not_mask);
2755   orr(tmp, tmp, new_val);
2756   sc_w(tmp, tmp, aligned_addr, release);
2757   bnez(tmp, fail);
2758 
2759   // Success
2760   mv(result, 1);
2761   j(done);
2762 
2763   // Fail
2764   bind(fail);
2765   mv(result, zr);
2766 
2767   bind(done);
2768 }
2769 
2770 void MacroAssembler::cmpxchg(Register addr, Register expected,
2771                              Register new_val,
2772                              enum operand_size size,
2773                              Assembler::Aqrl acquire, Assembler::Aqrl release,
2774                              Register result, bool result_as_bool) {
2775   assert(size != int8 && size != int16, "unsupported operand size");
2776   assert_different_registers(addr, t0);
2777   assert_different_registers(expected, t0);
2778   assert_different_registers(new_val, t0);
2779 
2780   Label retry_load, done, ne_done;
2781   bind(retry_load);
2782   load_reserved(addr, size, acquire);
2783   bne(t0, expected, ne_done);
2784   store_conditional(addr, new_val, size, release);
2785   bnez(t0, retry_load);
2786 
2787   // equal, succeed
2788   if (result_as_bool) {
2789     mv(result, 1);
2790   } else {
2791     mv(result, expected);
2792   }
2793   j(done);
2794 
2795   // not equal, failed
2796   bind(ne_done);
2797   if (result_as_bool) {
2798     mv(result, zr);
2799   } else {
2800     mv(result, t0);
2801   }
2802 
2803   bind(done);
2804 }
2805 
2806 void MacroAssembler::cmpxchg_weak(Register addr, Register expected,
2807                                   Register new_val,
2808                                   enum operand_size size,
2809                                   Assembler::Aqrl acquire, Assembler::Aqrl release,
2810                                   Register result) {
2811   assert_different_registers(addr, t0);
2812   assert_different_registers(expected, t0);
2813   assert_different_registers(new_val, t0);
2814 
2815   Label fail, done;
2816   load_reserved(addr, size, acquire);
2817   bne(t0, expected, fail);
2818   store_conditional(addr, new_val, size, release);
2819   bnez(t0, fail);
2820 
2821   // Success
2822   mv(result, 1);
2823   j(done);
2824 
2825   // Fail
2826   bind(fail);
2827   mv(result, zr);
2828 
2829   bind(done);
2830 }
2831 
2832 #define ATOMIC_OP(NAME, AOP, ACQUIRE, RELEASE)                                              \
2833 void MacroAssembler::atomic_##NAME(Register prev, RegisterOrConstant incr, Register addr) { \
2834   prev = prev->is_valid() ? prev : zr;                                                      \
2835   if (incr.is_register()) {                                                                 \
2836     AOP(prev, addr, incr.as_register(), (Assembler::Aqrl)(ACQUIRE | RELEASE));              \
2837   } else {                                                                                  \
2838     mv(t0, incr.as_constant());                                                             \
2839     AOP(prev, addr, t0, (Assembler::Aqrl)(ACQUIRE | RELEASE));                              \
2840   }                                                                                         \
2841   return;                                                                                   \
2842 }
2843 
2844 ATOMIC_OP(add, amoadd_d, Assembler::relaxed, Assembler::relaxed)
2845 ATOMIC_OP(addw, amoadd_w, Assembler::relaxed, Assembler::relaxed)
2846 ATOMIC_OP(addal, amoadd_d, Assembler::aq, Assembler::rl)
2847 ATOMIC_OP(addalw, amoadd_w, Assembler::aq, Assembler::rl)
2848 
2849 #undef ATOMIC_OP
2850 
2851 #define ATOMIC_XCHG(OP, AOP, ACQUIRE, RELEASE)                                       \
2852 void MacroAssembler::atomic_##OP(Register prev, Register newv, Register addr) {      \
2853   prev = prev->is_valid() ? prev : zr;                                               \
2854   AOP(prev, addr, newv, (Assembler::Aqrl)(ACQUIRE | RELEASE));                       \
2855   return;                                                                            \
2856 }
2857 
2858 ATOMIC_XCHG(xchg, amoswap_d, Assembler::relaxed, Assembler::relaxed)
2859 ATOMIC_XCHG(xchgw, amoswap_w, Assembler::relaxed, Assembler::relaxed)
2860 ATOMIC_XCHG(xchgal, amoswap_d, Assembler::aq, Assembler::rl)
2861 ATOMIC_XCHG(xchgalw, amoswap_w, Assembler::aq, Assembler::rl)
2862 
2863 #undef ATOMIC_XCHG
2864 
2865 #define ATOMIC_XCHGU(OP1, OP2)                                                       \
2866 void MacroAssembler::atomic_##OP1(Register prev, Register newv, Register addr) {     \
2867   atomic_##OP2(prev, newv, addr);                                                    \
2868   zero_extend(prev, prev, 32);                                                       \
2869   return;                                                                            \
2870 }
2871 
2872 ATOMIC_XCHGU(xchgwu, xchgw)
2873 ATOMIC_XCHGU(xchgalwu, xchgalw)
2874 
2875 #undef ATOMIC_XCHGU
2876 
2877 void MacroAssembler::far_jump(Address entry, Register tmp) {
2878   assert(ReservedCodeCacheSize < 4*G, "branch out of range");
2879   assert(CodeCache::find_blob(entry.target()) != nullptr,
2880          "destination of far call not found in code cache");
2881   assert(entry.rspec().type() == relocInfo::external_word_type
2882         || entry.rspec().type() == relocInfo::runtime_call_type
2883         || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type");
2884   IncompressibleRegion ir(this);  // Fixed length: see MacroAssembler::far_branch_size()
2885   if (far_branches()) {
2886     // We can use auipc + jalr here because we know that the total size of
2887     // the code cache cannot exceed 2Gb.
2888     relocate(entry.rspec(), [&] {
2889       int32_t offset;
2890       la_patchable(tmp, entry, offset);
2891       jalr(x0, tmp, offset);
2892     });
2893   } else {
2894     j(entry);
2895   }
2896 }
2897 
2898 void MacroAssembler::far_call(Address entry, Register tmp) {
2899   assert(ReservedCodeCacheSize < 4*G, "branch out of range");
2900   assert(CodeCache::find_blob(entry.target()) != nullptr,
2901          "destination of far call not found in code cache");
2902   assert(entry.rspec().type() == relocInfo::external_word_type
2903         || entry.rspec().type() == relocInfo::runtime_call_type
2904         || entry.rspec().type() == relocInfo::none, "wrong entry relocInfo type");
2905   IncompressibleRegion ir(this);  // Fixed length: see MacroAssembler::far_branch_size()
2906   if (far_branches()) {
2907     // We can use auipc + jalr here because we know that the total size of
2908     // the code cache cannot exceed 2Gb.
2909     relocate(entry.rspec(), [&] {
2910       int32_t offset;
2911       la_patchable(tmp, entry, offset);
2912       jalr(x1, tmp, offset); // link
2913     });
2914   } else {
2915     jal(entry); // link
2916   }
2917 }
2918 
2919 void MacroAssembler::check_klass_subtype_fast_path(Register sub_klass,
2920                                                    Register super_klass,
2921                                                    Register tmp_reg,
2922                                                    Label* L_success,
2923                                                    Label* L_failure,
2924                                                    Label* L_slow_path,
2925                                                    Register super_check_offset) {
2926   assert_different_registers(sub_klass, super_klass, tmp_reg);
2927   bool must_load_sco = (super_check_offset == noreg);
2928   if (must_load_sco) {
2929     assert(tmp_reg != noreg, "supply either a temp or a register offset");
2930   } else {
2931     assert_different_registers(sub_klass, super_klass, super_check_offset);
2932   }
2933 
2934   Label L_fallthrough;
2935   int label_nulls = 0;
2936   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
2937   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
2938   if (L_slow_path == nullptr) { L_slow_path = &L_fallthrough; label_nulls++; }
2939   assert(label_nulls <= 1, "at most one null in batch");
2940 
2941   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
2942   int sco_offset = in_bytes(Klass::super_check_offset_offset());
2943   Address super_check_offset_addr(super_klass, sco_offset);
2944 
2945   // Hacked jmp, which may only be used just before L_fallthrough.
2946 #define final_jmp(label)                                                \
2947   if (&(label) == &L_fallthrough) { /*do nothing*/ }                    \
2948   else                            j(label)             /*omit semi*/
2949 
2950   // If the pointers are equal, we are done (e.g., String[] elements).
2951   // This self-check enables sharing of secondary supertype arrays among
2952   // non-primary types such as array-of-interface. Otherwise, each such
2953   // type would need its own customized SSA.
2954   // We move this check to the front of the fast path because many
2955   // type checks are in fact trivially successful in this manner,
2956   // so we get a nicely predicted branch right at the start of the check.
2957   beq(sub_klass, super_klass, *L_success);
2958 
2959   // Check the supertype display:
2960   if (must_load_sco) {
2961     lwu(tmp_reg, super_check_offset_addr);
2962     super_check_offset = tmp_reg;
2963   }
2964   add(t0, sub_klass, super_check_offset);
2965   Address super_check_addr(t0);
2966   ld(t0, super_check_addr); // load displayed supertype
2967 
2968   // This check has worked decisively for primary supers.
2969   // Secondary supers are sought in the super_cache ('super_cache_addr').
2970   // (Secondary supers are interfaces and very deeply nested subtypes.)
2971   // This works in the same check above because of a tricky aliasing
2972   // between the super_Cache and the primary super display elements.
2973   // (The 'super_check_addr' can address either, as the case requires.)
2974   // Note that the cache is updated below if it does not help us find
2975   // what we need immediately.
2976   // So if it was a primary super, we can just fail immediately.
2977   // Otherwise, it's the slow path for us (no success at this point).
2978 
2979   beq(super_klass, t0, *L_success);
2980   mv(t1, sc_offset);
2981   if (L_failure == &L_fallthrough) {
2982     beq(super_check_offset, t1, *L_slow_path);
2983   } else {
2984     bne(super_check_offset, t1, *L_failure, /* is_far */ true);
2985     final_jmp(*L_slow_path);
2986   }
2987 
2988   bind(L_fallthrough);
2989 
2990 #undef final_jmp
2991 }
2992 
2993 // Scans count pointer sized words at [addr] for occurrence of value,
2994 // generic
2995 void MacroAssembler::repne_scan(Register addr, Register value, Register count,
2996                                 Register tmp) {
2997   Label Lloop, Lexit;
2998   beqz(count, Lexit);
2999   bind(Lloop);
3000   ld(tmp, addr);
3001   beq(value, tmp, Lexit);
3002   add(addr, addr, wordSize);
3003   sub(count, count, 1);
3004   bnez(count, Lloop);
3005   bind(Lexit);
3006 }
3007 
3008 void MacroAssembler::check_klass_subtype_slow_path(Register sub_klass,
3009                                                    Register super_klass,
3010                                                    Register tmp1_reg,
3011                                                    Register tmp2_reg,
3012                                                    Label* L_success,
3013                                                    Label* L_failure) {
3014   assert_different_registers(sub_klass, super_klass, tmp1_reg);
3015   if (tmp2_reg != noreg) {
3016     assert_different_registers(sub_klass, super_klass, tmp1_reg, tmp2_reg, t0);
3017   }
3018 #define IS_A_TEMP(reg) ((reg) == tmp1_reg || (reg) == tmp2_reg)
3019 
3020   Label L_fallthrough;
3021   int label_nulls = 0;
3022   if (L_success == nullptr)   { L_success   = &L_fallthrough; label_nulls++; }
3023   if (L_failure == nullptr)   { L_failure   = &L_fallthrough; label_nulls++; }
3024 
3025   assert(label_nulls <= 1, "at most one null in the batch");
3026 
3027   // A couple of useful fields in sub_klass:
3028   int ss_offset = in_bytes(Klass::secondary_supers_offset());
3029   int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
3030   Address secondary_supers_addr(sub_klass, ss_offset);
3031   Address super_cache_addr(     sub_klass, sc_offset);
3032 
3033   BLOCK_COMMENT("check_klass_subtype_slow_path");
3034 
3035   // Do a linear scan of the secondary super-klass chain.
3036   // This code is rarely used, so simplicity is a virtue here.
3037   // The repne_scan instruction uses fixed registers, which we must spill.
3038   // Don't worry too much about pre-existing connections with the input regs.
3039 
3040   assert(sub_klass != x10, "killed reg"); // killed by mv(x10, super)
3041   assert(sub_klass != x12, "killed reg"); // killed by la(x12, &pst_counter)
3042 
3043   RegSet pushed_registers;
3044   if (!IS_A_TEMP(x12)) {
3045     pushed_registers += x12;
3046   }
3047   if (!IS_A_TEMP(x15)) {
3048     pushed_registers += x15;
3049   }
3050 
3051   if (super_klass != x10) {
3052     if (!IS_A_TEMP(x10)) {
3053       pushed_registers += x10;
3054     }
3055   }
3056 
3057   push_reg(pushed_registers, sp);
3058 
3059   // Get super_klass value into x10 (even if it was in x15 or x12)
3060   mv(x10, super_klass);
3061 
3062 #ifndef PRODUCT
3063   mv(t1, (address)&SharedRuntime::_partial_subtype_ctr);
3064   Address pst_counter_addr(t1);
3065   ld(t0, pst_counter_addr);
3066   add(t0, t0, 1);
3067   sd(t0, pst_counter_addr);
3068 #endif // PRODUCT
3069 
3070   // We will consult the secondary-super array.
3071   ld(x15, secondary_supers_addr);
3072   // Load the array length.
3073   lwu(x12, Address(x15, Array<Klass*>::length_offset_in_bytes()));
3074   // Skip to start of data.
3075   add(x15, x15, Array<Klass*>::base_offset_in_bytes());
3076 
3077   // Set t0 to an obvious invalid value, falling through by default
3078   mv(t0, -1);
3079   // Scan X12 words at [X15] for an occurrence of X10.
3080   repne_scan(x15, x10, x12, t0);
3081 
3082   // pop will restore x10, so we should use a temp register to keep its value
3083   mv(t1, x10);
3084 
3085   // Unspill the temp registers:
3086   pop_reg(pushed_registers, sp);
3087 
3088   bne(t1, t0, *L_failure);
3089 
3090   // Success. Cache the super we found an proceed in triumph.
3091   sd(super_klass, super_cache_addr);
3092 
3093   if (L_success != &L_fallthrough) {
3094     j(*L_success);
3095   }
3096 
3097 #undef IS_A_TEMP
3098 
3099   bind(L_fallthrough);
3100 }
3101 
3102 // Defines obj, preserves var_size_in_bytes, okay for tmp2 == var_size_in_bytes.
3103 void MacroAssembler::tlab_allocate(Register obj,
3104                                    Register var_size_in_bytes,
3105                                    int con_size_in_bytes,
3106                                    Register tmp1,
3107                                    Register tmp2,
3108                                    Label& slow_case,
3109                                    bool is_far) {
3110   BarrierSetAssembler *bs = BarrierSet::barrier_set()->barrier_set_assembler();
3111   bs->tlab_allocate(this, obj, var_size_in_bytes, con_size_in_bytes, tmp1, tmp2, slow_case, is_far);
3112 }
3113 
3114 // get_thread() can be called anywhere inside generated code so we
3115 // need to save whatever non-callee save context might get clobbered
3116 // by the call to Thread::current() or, indeed, the call setup code.
3117 void MacroAssembler::get_thread(Register thread) {
3118   // save all call-clobbered regs except thread
3119   RegSet saved_regs = RegSet::range(x5, x7) + RegSet::range(x10, x17) +
3120                       RegSet::range(x28, x31) + ra - thread;
3121   push_reg(saved_regs, sp);
3122 
3123   mv(ra, CAST_FROM_FN_PTR(address, Thread::current));
3124   jalr(ra);
3125   if (thread != c_rarg0) {
3126     mv(thread, c_rarg0);
3127   }
3128 
3129   // restore pushed registers
3130   pop_reg(saved_regs, sp);
3131 }
3132 
3133 void MacroAssembler::load_byte_map_base(Register reg) {
3134   CardTable::CardValue* byte_map_base =
3135     ((CardTableBarrierSet*)(BarrierSet::barrier_set()))->card_table()->byte_map_base();
3136   mv(reg, (uint64_t)byte_map_base);
3137 }
3138 
3139 void MacroAssembler::la_patchable(Register reg1, const Address &dest, int32_t &offset) {
3140   unsigned long low_address = (uintptr_t)CodeCache::low_bound();
3141   unsigned long high_address = (uintptr_t)CodeCache::high_bound();
3142   unsigned long dest_address = (uintptr_t)dest.target();
3143   long offset_low = dest_address - low_address;
3144   long offset_high = dest_address - high_address;
3145 
3146   assert(dest.getMode() == Address::literal, "la_patchable must be applied to a literal address");
3147   assert((uintptr_t)dest.target() < (1ull << 48), "bad address");
3148 
3149   // RISC-V doesn't compute a page-aligned address, in order to partially
3150   // compensate for the use of *signed* offsets in its base+disp12
3151   // addressing mode (RISC-V's PC-relative reach remains asymmetric
3152   // [-(2G + 2K), 2G - 2K).
3153   if (offset_high >= -((1L << 31) + (1L << 11)) && offset_low < (1L << 31) - (1L << 11)) {
3154     int64_t distance = dest.target() - pc();
3155     auipc(reg1, (int32_t)distance + 0x800);
3156     offset = ((int32_t)distance << 20) >> 20;
3157   } else {
3158     movptr(reg1, dest.target(), offset);
3159   }
3160 }
3161 
3162 void MacroAssembler::build_frame(int framesize) {
3163   assert(framesize >= 2, "framesize must include space for FP/RA");
3164   assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
3165   sub(sp, sp, framesize);
3166   sd(fp, Address(sp, framesize - 2 * wordSize));
3167   sd(ra, Address(sp, framesize - wordSize));
3168   if (PreserveFramePointer) { add(fp, sp, framesize); }
3169 }
3170 
3171 void MacroAssembler::remove_frame(int framesize) {
3172   assert(framesize >= 2, "framesize must include space for FP/RA");
3173   assert(framesize % (2*wordSize) == 0, "must preserve 2*wordSize alignment");
3174   ld(fp, Address(sp, framesize - 2 * wordSize));
3175   ld(ra, Address(sp, framesize - wordSize));
3176   add(sp, sp, framesize);
3177 }
3178 
3179 void MacroAssembler::reserved_stack_check() {
3180     // testing if reserved zone needs to be enabled
3181     Label no_reserved_zone_enabling;
3182 
3183     ld(t0, Address(xthread, JavaThread::reserved_stack_activation_offset()));
3184     bltu(sp, t0, no_reserved_zone_enabling);
3185 
3186     enter();   // RA and FP are live.
3187     mv(c_rarg0, xthread);
3188     RuntimeAddress target(CAST_FROM_FN_PTR(address, SharedRuntime::enable_stack_reserved_zone));
3189     relocate(target.rspec(), [&] {
3190       int32_t offset;
3191       la_patchable(t0, target, offset);
3192       jalr(x1, t0, offset);
3193     });
3194     leave();
3195 
3196     // We have already removed our own frame.
3197     // throw_delayed_StackOverflowError will think that it's been
3198     // called by our caller.
3199     target = RuntimeAddress(StubRoutines::throw_delayed_StackOverflowError_entry());
3200     relocate(target.rspec(), [&] {
3201       int32_t offset;
3202       la_patchable(t0, target, offset);
3203       jalr(x0, t0, offset);
3204     });
3205     should_not_reach_here();
3206 
3207     bind(no_reserved_zone_enabling);
3208 }
3209 
3210 // Move the address of the polling page into dest.
3211 void MacroAssembler::get_polling_page(Register dest, relocInfo::relocType rtype) {
3212   ld(dest, Address(xthread, JavaThread::polling_page_offset()));
3213 }
3214 
3215 // Read the polling page.  The address of the polling page must
3216 // already be in r.
3217 void MacroAssembler::read_polling_page(Register r, int32_t offset, relocInfo::relocType rtype) {
3218   relocate(rtype, [&] {
3219     lwu(zr, Address(r, offset));
3220   });
3221 }
3222 
3223 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
3224 #ifdef ASSERT
3225   {
3226     ThreadInVMfromUnknown tiv;
3227     assert (UseCompressedOops, "should only be used for compressed oops");
3228     assert (Universe::heap() != nullptr, "java heap should be initialized");
3229     assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
3230     assert(Universe::heap()->is_in(JNIHandles::resolve(obj)), "should be real oop");
3231   }
3232 #endif
3233   int oop_index = oop_recorder()->find_index(obj);
3234   relocate(oop_Relocation::spec(oop_index), [&] {
3235     li32(dst, 0xDEADBEEF);
3236   });
3237   zero_extend(dst, dst, 32);
3238 }
3239 
3240 void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
3241   assert (UseCompressedClassPointers, "should only be used for compressed headers");
3242   assert (oop_recorder() != nullptr, "this assembler needs an OopRecorder");
3243   int index = oop_recorder()->find_index(k);
3244   assert(!Universe::heap()->is_in(k), "should not be an oop");
3245 
3246   narrowKlass nk = CompressedKlassPointers::encode(k);
3247   relocate(metadata_Relocation::spec(index), [&] {
3248     li32(dst, nk);
3249   });
3250   zero_extend(dst, dst, 32);
3251 }
3252 
3253 // Maybe emit a call via a trampoline. If the code cache is small
3254 // trampolines won't be emitted.
3255 address MacroAssembler::trampoline_call(Address entry) {
3256   assert(entry.rspec().type() == relocInfo::runtime_call_type ||
3257          entry.rspec().type() == relocInfo::opt_virtual_call_type ||
3258          entry.rspec().type() == relocInfo::static_call_type ||
3259          entry.rspec().type() == relocInfo::virtual_call_type, "wrong reloc type");
3260 
3261   address target = entry.target();
3262 
3263   // We need a trampoline if branches are far.
3264   if (far_branches()) {
3265     if (!in_scratch_emit_size()) {
3266       if (entry.rspec().type() == relocInfo::runtime_call_type) {
3267         assert(CodeBuffer::supports_shared_stubs(), "must support shared stubs");
3268         code()->share_trampoline_for(entry.target(), offset());
3269       } else {
3270         address stub = emit_trampoline_stub(offset(), target);
3271         if (stub == nullptr) {
3272           postcond(pc() == badAddress);
3273           return nullptr; // CodeCache is full
3274         }
3275       }
3276     }
3277     target = pc();
3278   }
3279 
3280   address call_pc = pc();
3281 #ifdef ASSERT
3282   if (entry.rspec().type() != relocInfo::runtime_call_type) {
3283     assert_alignment(call_pc);
3284   }
3285 #endif
3286   relocate(entry.rspec(), [&] {
3287     jal(target);
3288   });
3289 
3290   postcond(pc() != badAddress);
3291   return call_pc;
3292 }
3293 
3294 address MacroAssembler::ic_call(address entry, jint method_index) {
3295   RelocationHolder rh = virtual_call_Relocation::spec(pc(), method_index);
3296   IncompressibleRegion ir(this);  // relocations
3297   movptr(t1, (address)Universe::non_oop_word());
3298   assert_cond(entry != nullptr);
3299   return trampoline_call(Address(entry, rh));
3300 }
3301 
3302 // Emit a trampoline stub for a call to a target which is too far away.
3303 //
3304 // code sequences:
3305 //
3306 // call-site:
3307 //   branch-and-link to <destination> or <trampoline stub>
3308 //
3309 // Related trampoline stub for this call site in the stub section:
3310 //   load the call target from the constant pool
3311 //   branch (RA still points to the call site above)
3312 
3313 address MacroAssembler::emit_trampoline_stub(int insts_call_instruction_offset,
3314                                              address dest) {
3315   // Max stub size: alignment nop, TrampolineStub.
3316   address stub = start_a_stub(max_trampoline_stub_size());
3317   if (stub == nullptr) {
3318     return nullptr;  // CodeBuffer::expand failed
3319   }
3320 
3321   // We are always 4-byte aligned here.
3322   assert_alignment(pc());
3323 
3324   // Create a trampoline stub relocation which relates this trampoline stub
3325   // with the call instruction at insts_call_instruction_offset in the
3326   // instructions code-section.
3327 
3328   // Make sure the address of destination 8-byte aligned after 3 instructions.
3329   align(wordSize, NativeCallTrampolineStub::data_offset);
3330 
3331   RelocationHolder rh = trampoline_stub_Relocation::spec(code()->insts()->start() +
3332                                                          insts_call_instruction_offset);
3333   const int stub_start_offset = offset();
3334   relocate(rh, [&] {
3335     // Now, create the trampoline stub's code:
3336     // - load the call
3337     // - call
3338     Label target;
3339     ld(t0, target);  // auipc + ld
3340     jr(t0);          // jalr
3341     bind(target);
3342     assert(offset() - stub_start_offset == NativeCallTrampolineStub::data_offset,
3343            "should be");
3344     assert(offset() % wordSize == 0, "bad alignment");
3345     emit_int64((int64_t)dest);
3346   });
3347 
3348   const address stub_start_addr = addr_at(stub_start_offset);
3349 
3350   assert(is_NativeCallTrampolineStub_at(stub_start_addr), "doesn't look like a trampoline");
3351 
3352   end_a_stub();
3353   return stub_start_addr;
3354 }
3355 
3356 int MacroAssembler::max_trampoline_stub_size() {
3357   // Max stub size: alignment nop, TrampolineStub.
3358   return NativeInstruction::instruction_size + NativeCallTrampolineStub::instruction_size;
3359 }
3360 
3361 int MacroAssembler::static_call_stub_size() {
3362   // (lui, addi, slli, addi, slli, addi) + (lui, addi, slli, addi, slli) + jalr
3363   return 12 * NativeInstruction::instruction_size;
3364 }
3365 
3366 Address MacroAssembler::add_memory_helper(const Address dst, Register tmp) {
3367   switch (dst.getMode()) {
3368     case Address::base_plus_offset:
3369       // This is the expected mode, although we allow all the other
3370       // forms below.
3371       return form_address(tmp, dst.base(), dst.offset());
3372     default:
3373       la(tmp, dst);
3374       return Address(tmp);
3375   }
3376 }
3377 
3378 void MacroAssembler::increment(const Address dst, int64_t value, Register tmp1, Register tmp2) {
3379   assert(((dst.getMode() == Address::base_plus_offset &&
3380            is_simm12(dst.offset())) || is_simm12(value)),
3381           "invalid value and address mode combination");
3382   Address adr = add_memory_helper(dst, tmp2);
3383   assert(!adr.uses(tmp1), "invalid dst for address increment");
3384   ld(tmp1, adr);
3385   add(tmp1, tmp1, value, tmp2);
3386   sd(tmp1, adr);
3387 }
3388 
3389 void MacroAssembler::incrementw(const Address dst, int32_t value, Register tmp1, Register tmp2) {
3390   assert(((dst.getMode() == Address::base_plus_offset &&
3391            is_simm12(dst.offset())) || is_simm12(value)),
3392           "invalid value and address mode combination");
3393   Address adr = add_memory_helper(dst, tmp2);
3394   assert(!adr.uses(tmp1), "invalid dst for address increment");
3395   lwu(tmp1, adr);
3396   addw(tmp1, tmp1, value, tmp2);
3397   sw(tmp1, adr);
3398 }
3399 
3400 void MacroAssembler::decrement(const Address dst, int64_t value, Register tmp1, Register tmp2) {
3401   assert(((dst.getMode() == Address::base_plus_offset &&
3402            is_simm12(dst.offset())) || is_simm12(value)),
3403           "invalid value and address mode combination");
3404   Address adr = add_memory_helper(dst, tmp2);
3405   assert(!adr.uses(tmp1), "invalid dst for address decrement");
3406   ld(tmp1, adr);
3407   sub(tmp1, tmp1, value, tmp2);
3408   sd(tmp1, adr);
3409 }
3410 
3411 void MacroAssembler::decrementw(const Address dst, int32_t value, Register tmp1, Register tmp2) {
3412   assert(((dst.getMode() == Address::base_plus_offset &&
3413            is_simm12(dst.offset())) || is_simm12(value)),
3414           "invalid value and address mode combination");
3415   Address adr = add_memory_helper(dst, tmp2);
3416   assert(!adr.uses(tmp1), "invalid dst for address decrement");
3417   lwu(tmp1, adr);
3418   subw(tmp1, tmp1, value, tmp2);
3419   sw(tmp1, adr);
3420 }
3421 
3422 void MacroAssembler::cmpptr(Register src1, Address src2, Label& equal) {
3423   assert_different_registers(src1, t0);
3424   relocate(src2.rspec(), [&] {
3425     int32_t offset;
3426     la_patchable(t0, src2, offset);
3427     ld(t0, Address(t0, offset));
3428   });
3429   beq(src1, t0, equal);
3430 }
3431 
3432 void MacroAssembler::load_method_holder_cld(Register result, Register method) {
3433   load_method_holder(result, method);
3434   ld(result, Address(result, InstanceKlass::class_loader_data_offset()));
3435 }
3436 
3437 void MacroAssembler::load_method_holder(Register holder, Register method) {
3438   ld(holder, Address(method, Method::const_offset()));                      // ConstMethod*
3439   ld(holder, Address(holder, ConstMethod::constants_offset()));             // ConstantPool*
3440   ld(holder, Address(holder, ConstantPool::pool_holder_offset()));          // InstanceKlass*
3441 }
3442 
3443 // string indexof
3444 // compute index by trailing zeros
3445 void MacroAssembler::compute_index(Register haystack, Register trailing_zeros,
3446                                    Register match_mask, Register result,
3447                                    Register ch2, Register tmp,
3448                                    bool haystack_isL) {
3449   int haystack_chr_shift = haystack_isL ? 0 : 1;
3450   srl(match_mask, match_mask, trailing_zeros);
3451   srli(match_mask, match_mask, 1);
3452   srli(tmp, trailing_zeros, LogBitsPerByte);
3453   if (!haystack_isL) andi(tmp, tmp, 0xE);
3454   add(haystack, haystack, tmp);
3455   ld(ch2, Address(haystack));
3456   if (!haystack_isL) srli(tmp, tmp, haystack_chr_shift);
3457   add(result, result, tmp);
3458 }
3459 
3460 // string indexof
3461 // Find pattern element in src, compute match mask,
3462 // only the first occurrence of 0x80/0x8000 at low bits is the valid match index
3463 // match mask patterns and corresponding indices would be like:
3464 // - 0x8080808080808080 (Latin1)
3465 // -   7 6 5 4 3 2 1 0  (match index)
3466 // - 0x8000800080008000 (UTF16)
3467 // -   3   2   1   0    (match index)
3468 void MacroAssembler::compute_match_mask(Register src, Register pattern, Register match_mask,
3469                                         Register mask1, Register mask2) {
3470   xorr(src, pattern, src);
3471   sub(match_mask, src, mask1);
3472   orr(src, src, mask2);
3473   notr(src, src);
3474   andr(match_mask, match_mask, src);
3475 }
3476 
3477 #ifdef COMPILER2
3478 // Code for BigInteger::mulAdd intrinsic
3479 // out     = x10
3480 // in      = x11
3481 // offset  = x12  (already out.length-offset)
3482 // len     = x13
3483 // k       = x14
3484 // tmp     = x28
3485 //
3486 // pseudo code from java implementation:
3487 // long kLong = k & LONG_MASK;
3488 // carry = 0;
3489 // offset = out.length-offset - 1;
3490 // for (int j = len - 1; j >= 0; j--) {
3491 //     product = (in[j] & LONG_MASK) * kLong + (out[offset] & LONG_MASK) + carry;
3492 //     out[offset--] = (int)product;
3493 //     carry = product >>> 32;
3494 // }
3495 // return (int)carry;
3496 void MacroAssembler::mul_add(Register out, Register in, Register offset,
3497                              Register len, Register k, Register tmp) {
3498   Label L_tail_loop, L_unroll, L_end;
3499   mv(tmp, out);
3500   mv(out, zr);
3501   blez(len, L_end);
3502   zero_extend(k, k, 32);
3503   slliw(t0, offset, LogBytesPerInt);
3504   add(offset, tmp, t0);
3505   slliw(t0, len, LogBytesPerInt);
3506   add(in, in, t0);
3507 
3508   const int unroll = 8;
3509   mv(tmp, unroll);
3510   blt(len, tmp, L_tail_loop);
3511   bind(L_unroll);
3512   for (int i = 0; i < unroll; i++) {
3513     sub(in, in, BytesPerInt);
3514     lwu(t0, Address(in, 0));
3515     mul(t1, t0, k);
3516     add(t0, t1, out);
3517     sub(offset, offset, BytesPerInt);
3518     lwu(t1, Address(offset, 0));
3519     add(t0, t0, t1);
3520     sw(t0, Address(offset, 0));
3521     srli(out, t0, 32);
3522   }
3523   subw(len, len, tmp);
3524   bge(len, tmp, L_unroll);
3525 
3526   bind(L_tail_loop);
3527   blez(len, L_end);
3528   sub(in, in, BytesPerInt);
3529   lwu(t0, Address(in, 0));
3530   mul(t1, t0, k);
3531   add(t0, t1, out);
3532   sub(offset, offset, BytesPerInt);
3533   lwu(t1, Address(offset, 0));
3534   add(t0, t0, t1);
3535   sw(t0, Address(offset, 0));
3536   srli(out, t0, 32);
3537   subw(len, len, 1);
3538   j(L_tail_loop);
3539 
3540   bind(L_end);
3541 }
3542 
3543 // add two unsigned input and output carry
3544 void MacroAssembler::cad(Register dst, Register src1, Register src2, Register carry)
3545 {
3546   assert_different_registers(dst, carry);
3547   assert_different_registers(dst, src2);
3548   add(dst, src1, src2);
3549   sltu(carry, dst, src2);
3550 }
3551 
3552 // add two input with carry
3553 void MacroAssembler::adc(Register dst, Register src1, Register src2, Register carry) {
3554   assert_different_registers(dst, carry);
3555   add(dst, src1, src2);
3556   add(dst, dst, carry);
3557 }
3558 
3559 // add two unsigned input with carry and output carry
3560 void MacroAssembler::cadc(Register dst, Register src1, Register src2, Register carry) {
3561   assert_different_registers(dst, src2);
3562   adc(dst, src1, src2, carry);
3563   sltu(carry, dst, src2);
3564 }
3565 
3566 void MacroAssembler::add2_with_carry(Register final_dest_hi, Register dest_hi, Register dest_lo,
3567                                      Register src1, Register src2, Register carry) {
3568   cad(dest_lo, dest_lo, src1, carry);
3569   add(dest_hi, dest_hi, carry);
3570   cad(dest_lo, dest_lo, src2, carry);
3571   add(final_dest_hi, dest_hi, carry);
3572 }
3573 
3574 /**
3575  * Multiply 32 bit by 32 bit first loop.
3576  */
3577 void MacroAssembler::multiply_32_x_32_loop(Register x, Register xstart, Register x_xstart,
3578                                            Register y, Register y_idx, Register z,
3579                                            Register carry, Register product,
3580                                            Register idx, Register kdx) {
3581   // jlong carry, x[], y[], z[];
3582   // for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) {
3583   //     long product = y[idx] * x[xstart] + carry;
3584   //     z[kdx] = (int)product;
3585   //     carry = product >>> 32;
3586   // }
3587   // z[xstart] = (int)carry;
3588 
3589   Label L_first_loop, L_first_loop_exit;
3590   blez(idx, L_first_loop_exit);
3591 
3592   shadd(t0, xstart, x, t0, LogBytesPerInt);
3593   lwu(x_xstart, Address(t0, 0));
3594 
3595   bind(L_first_loop);
3596   subw(idx, idx, 1);
3597   shadd(t0, idx, y, t0, LogBytesPerInt);
3598   lwu(y_idx, Address(t0, 0));
3599   mul(product, x_xstart, y_idx);
3600   add(product, product, carry);
3601   srli(carry, product, 32);
3602   subw(kdx, kdx, 1);
3603   shadd(t0, kdx, z, t0, LogBytesPerInt);
3604   sw(product, Address(t0, 0));
3605   bgtz(idx, L_first_loop);
3606 
3607   bind(L_first_loop_exit);
3608 }
3609 
3610 /**
3611  * Multiply 64 bit by 64 bit first loop.
3612  */
3613 void MacroAssembler::multiply_64_x_64_loop(Register x, Register xstart, Register x_xstart,
3614                                            Register y, Register y_idx, Register z,
3615                                            Register carry, Register product,
3616                                            Register idx, Register kdx) {
3617   //
3618   //  jlong carry, x[], y[], z[];
3619   //  for (int idx=ystart, kdx=ystart+1+xstart; idx >= 0; idx--, kdx--) {
3620   //    huge_128 product = y[idx] * x[xstart] + carry;
3621   //    z[kdx] = (jlong)product;
3622   //    carry  = (jlong)(product >>> 64);
3623   //  }
3624   //  z[xstart] = carry;
3625   //
3626 
3627   Label L_first_loop, L_first_loop_exit;
3628   Label L_one_x, L_one_y, L_multiply;
3629 
3630   subw(xstart, xstart, 1);
3631   bltz(xstart, L_one_x);
3632 
3633   shadd(t0, xstart, x, t0, LogBytesPerInt);
3634   ld(x_xstart, Address(t0, 0));
3635   ror_imm(x_xstart, x_xstart, 32); // convert big-endian to little-endian
3636 
3637   bind(L_first_loop);
3638   subw(idx, idx, 1);
3639   bltz(idx, L_first_loop_exit);
3640   subw(idx, idx, 1);
3641   bltz(idx, L_one_y);
3642 
3643   shadd(t0, idx, y, t0, LogBytesPerInt);
3644   ld(y_idx, Address(t0, 0));
3645   ror_imm(y_idx, y_idx, 32); // convert big-endian to little-endian
3646   bind(L_multiply);
3647 
3648   mulhu(t0, x_xstart, y_idx);
3649   mul(product, x_xstart, y_idx);
3650   cad(product, product, carry, t1);
3651   adc(carry, t0, zr, t1);
3652 
3653   subw(kdx, kdx, 2);
3654   ror_imm(product, product, 32); // back to big-endian
3655   shadd(t0, kdx, z, t0, LogBytesPerInt);
3656   sd(product, Address(t0, 0));
3657 
3658   j(L_first_loop);
3659 
3660   bind(L_one_y);
3661   lwu(y_idx, Address(y, 0));
3662   j(L_multiply);
3663 
3664   bind(L_one_x);
3665   lwu(x_xstart, Address(x, 0));
3666   j(L_first_loop);
3667 
3668   bind(L_first_loop_exit);
3669 }
3670 
3671 /**
3672  * Multiply 128 bit by 128 bit. Unrolled inner loop.
3673  *
3674  */
3675 void MacroAssembler::multiply_128_x_128_loop(Register y, Register z,
3676                                              Register carry, Register carry2,
3677                                              Register idx, Register jdx,
3678                                              Register yz_idx1, Register yz_idx2,
3679                                              Register tmp, Register tmp3, Register tmp4,
3680                                              Register tmp6, Register product_hi) {
3681   //   jlong carry, x[], y[], z[];
3682   //   int kdx = xstart+1;
3683   //   for (int idx=ystart-2; idx >= 0; idx -= 2) { // Third loop
3684   //     huge_128 tmp3 = (y[idx+1] * product_hi) + z[kdx+idx+1] + carry;
3685   //     jlong carry2  = (jlong)(tmp3 >>> 64);
3686   //     huge_128 tmp4 = (y[idx]   * product_hi) + z[kdx+idx] + carry2;
3687   //     carry  = (jlong)(tmp4 >>> 64);
3688   //     z[kdx+idx+1] = (jlong)tmp3;
3689   //     z[kdx+idx] = (jlong)tmp4;
3690   //   }
3691   //   idx += 2;
3692   //   if (idx > 0) {
3693   //     yz_idx1 = (y[idx] * product_hi) + z[kdx+idx] + carry;
3694   //     z[kdx+idx] = (jlong)yz_idx1;
3695   //     carry  = (jlong)(yz_idx1 >>> 64);
3696   //   }
3697   //
3698 
3699   Label L_third_loop, L_third_loop_exit, L_post_third_loop_done;
3700 
3701   srliw(jdx, idx, 2);
3702 
3703   bind(L_third_loop);
3704 
3705   subw(jdx, jdx, 1);
3706   bltz(jdx, L_third_loop_exit);
3707   subw(idx, idx, 4);
3708 
3709   shadd(t0, idx, y, t0, LogBytesPerInt);
3710   ld(yz_idx2, Address(t0, 0));
3711   ld(yz_idx1, Address(t0, wordSize));
3712 
3713   shadd(tmp6, idx, z, t0, LogBytesPerInt);
3714 
3715   ror_imm(yz_idx1, yz_idx1, 32); // convert big-endian to little-endian
3716   ror_imm(yz_idx2, yz_idx2, 32);
3717 
3718   ld(t1, Address(tmp6, 0));
3719   ld(t0, Address(tmp6, wordSize));
3720 
3721   mul(tmp3, product_hi, yz_idx1); //  yz_idx1 * product_hi -> tmp4:tmp3
3722   mulhu(tmp4, product_hi, yz_idx1);
3723 
3724   ror_imm(t0, t0, 32, tmp); // convert big-endian to little-endian
3725   ror_imm(t1, t1, 32, tmp);
3726 
3727   mul(tmp, product_hi, yz_idx2); //  yz_idx2 * product_hi -> carry2:tmp
3728   mulhu(carry2, product_hi, yz_idx2);
3729 
3730   cad(tmp3, tmp3, carry, carry);
3731   adc(tmp4, tmp4, zr, carry);
3732   cad(tmp3, tmp3, t0, t0);
3733   cadc(tmp4, tmp4, tmp, t0);
3734   adc(carry, carry2, zr, t0);
3735   cad(tmp4, tmp4, t1, carry2);
3736   adc(carry, carry, zr, carry2);
3737 
3738   ror_imm(tmp3, tmp3, 32); // convert little-endian to big-endian
3739   ror_imm(tmp4, tmp4, 32);
3740   sd(tmp4, Address(tmp6, 0));
3741   sd(tmp3, Address(tmp6, wordSize));
3742 
3743   j(L_third_loop);
3744 
3745   bind(L_third_loop_exit);
3746 
3747   andi(idx, idx, 0x3);
3748   beqz(idx, L_post_third_loop_done);
3749 
3750   Label L_check_1;
3751   subw(idx, idx, 2);
3752   bltz(idx, L_check_1);
3753 
3754   shadd(t0, idx, y, t0, LogBytesPerInt);
3755   ld(yz_idx1, Address(t0, 0));
3756   ror_imm(yz_idx1, yz_idx1, 32);
3757 
3758   mul(tmp3, product_hi, yz_idx1); //  yz_idx1 * product_hi -> tmp4:tmp3
3759   mulhu(tmp4, product_hi, yz_idx1);
3760 
3761   shadd(t0, idx, z, t0, LogBytesPerInt);
3762   ld(yz_idx2, Address(t0, 0));
3763   ror_imm(yz_idx2, yz_idx2, 32, tmp);
3764 
3765   add2_with_carry(carry, tmp4, tmp3, carry, yz_idx2, tmp);
3766 
3767   ror_imm(tmp3, tmp3, 32, tmp);
3768   sd(tmp3, Address(t0, 0));
3769 
3770   bind(L_check_1);
3771 
3772   andi(idx, idx, 0x1);
3773   subw(idx, idx, 1);
3774   bltz(idx, L_post_third_loop_done);
3775   shadd(t0, idx, y, t0, LogBytesPerInt);
3776   lwu(tmp4, Address(t0, 0));
3777   mul(tmp3, tmp4, product_hi); //  tmp4 * product_hi -> carry2:tmp3
3778   mulhu(carry2, tmp4, product_hi);
3779 
3780   shadd(t0, idx, z, t0, LogBytesPerInt);
3781   lwu(tmp4, Address(t0, 0));
3782 
3783   add2_with_carry(carry2, carry2, tmp3, tmp4, carry, t0);
3784 
3785   shadd(t0, idx, z, t0, LogBytesPerInt);
3786   sw(tmp3, Address(t0, 0));
3787 
3788   slli(t0, carry2, 32);
3789   srli(carry, tmp3, 32);
3790   orr(carry, carry, t0);
3791 
3792   bind(L_post_third_loop_done);
3793 }
3794 
3795 /**
3796  * Code for BigInteger::multiplyToLen() intrinsic.
3797  *
3798  * x10: x
3799  * x11: xlen
3800  * x12: y
3801  * x13: ylen
3802  * x14: z
3803  * x15: zlen
3804  * x16: tmp1
3805  * x17: tmp2
3806  * x7:  tmp3
3807  * x28: tmp4
3808  * x29: tmp5
3809  * x30: tmp6
3810  * x31: tmp7
3811  */
3812 void MacroAssembler::multiply_to_len(Register x, Register xlen, Register y, Register ylen,
3813                                      Register z, Register zlen,
3814                                      Register tmp1, Register tmp2, Register tmp3, Register tmp4,
3815                                      Register tmp5, Register tmp6, Register product_hi) {
3816   assert_different_registers(x, xlen, y, ylen, z, zlen, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6);
3817 
3818   const Register idx = tmp1;
3819   const Register kdx = tmp2;
3820   const Register xstart = tmp3;
3821 
3822   const Register y_idx = tmp4;
3823   const Register carry = tmp5;
3824   const Register product = xlen;
3825   const Register x_xstart = zlen; // reuse register
3826 
3827   mv(idx, ylen); // idx = ylen;
3828   mv(kdx, zlen); // kdx = xlen+ylen;
3829   mv(carry, zr); // carry = 0;
3830 
3831   Label L_multiply_64_x_64_loop, L_done;
3832 
3833   subw(xstart, xlen, 1);
3834   bltz(xstart, L_done);
3835 
3836   const Register jdx = tmp1;
3837 
3838   if (AvoidUnalignedAccesses) {
3839     // Check if x and y are both 8-byte aligned.
3840     orr(t0, xlen, ylen);
3841     test_bit(t0, t0, 0);
3842     beqz(t0, L_multiply_64_x_64_loop);
3843 
3844     multiply_32_x_32_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
3845     shadd(t0, xstart, z, t0, LogBytesPerInt);
3846     sw(carry, Address(t0, 0));
3847 
3848     Label L_second_loop_unaligned;
3849     bind(L_second_loop_unaligned);
3850     mv(carry, zr);
3851     mv(jdx, ylen);
3852     subw(xstart, xstart, 1);
3853     bltz(xstart, L_done);
3854     sub(sp, sp, 2 * wordSize);
3855     sd(z, Address(sp, 0));
3856     sd(zr, Address(sp, wordSize));
3857     shadd(t0, xstart, z, t0, LogBytesPerInt);
3858     addi(z, t0, 4);
3859     shadd(t0, xstart, x, t0, LogBytesPerInt);
3860     lwu(product, Address(t0, 0));
3861     Label L_third_loop, L_third_loop_exit;
3862 
3863     blez(jdx, L_third_loop_exit);
3864 
3865     bind(L_third_loop);
3866     subw(jdx, jdx, 1);
3867     shadd(t0, jdx, y, t0, LogBytesPerInt);
3868     lwu(t0, Address(t0, 0));
3869     mul(t1, t0, product);
3870     add(t0, t1, carry);
3871     shadd(tmp6, jdx, z, t1, LogBytesPerInt);
3872     lwu(t1, Address(tmp6, 0));
3873     add(t0, t0, t1);
3874     sw(t0, Address(tmp6, 0));
3875     srli(carry, t0, 32);
3876     bgtz(jdx, L_third_loop);
3877 
3878     bind(L_third_loop_exit);
3879     ld(z, Address(sp, 0));
3880     addi(sp, sp, 2 * wordSize);
3881     shadd(t0, xstart, z, t0, LogBytesPerInt);
3882     sw(carry, Address(t0, 0));
3883 
3884     j(L_second_loop_unaligned);
3885   }
3886 
3887   bind(L_multiply_64_x_64_loop);
3888   multiply_64_x_64_loop(x, xstart, x_xstart, y, y_idx, z, carry, product, idx, kdx);
3889 
3890   Label L_second_loop_aligned;
3891   beqz(kdx, L_second_loop_aligned);
3892 
3893   Label L_carry;
3894   subw(kdx, kdx, 1);
3895   beqz(kdx, L_carry);
3896 
3897   shadd(t0, kdx, z, t0, LogBytesPerInt);
3898   sw(carry, Address(t0, 0));
3899   srli(carry, carry, 32);
3900   subw(kdx, kdx, 1);
3901 
3902   bind(L_carry);
3903   shadd(t0, kdx, z, t0, LogBytesPerInt);
3904   sw(carry, Address(t0, 0));
3905 
3906   // Second and third (nested) loops.
3907   //
3908   // for (int i = xstart-1; i >= 0; i--) { // Second loop
3909   //   carry = 0;
3910   //   for (int jdx=ystart, k=ystart+1+i; jdx >= 0; jdx--, k--) { // Third loop
3911   //     long product = (y[jdx] & LONG_MASK) * (x[i] & LONG_MASK) +
3912   //                    (z[k] & LONG_MASK) + carry;
3913   //     z[k] = (int)product;
3914   //     carry = product >>> 32;
3915   //   }
3916   //   z[i] = (int)carry;
3917   // }
3918   //
3919   // i = xlen, j = tmp1, k = tmp2, carry = tmp5, x[i] = product_hi
3920 
3921   bind(L_second_loop_aligned);
3922   mv(carry, zr); // carry = 0;
3923   mv(jdx, ylen); // j = ystart+1
3924 
3925   subw(xstart, xstart, 1); // i = xstart-1;
3926   bltz(xstart, L_done);
3927 
3928   sub(sp, sp, 4 * wordSize);
3929   sd(z, Address(sp, 0));
3930 
3931   Label L_last_x;
3932   shadd(t0, xstart, z, t0, LogBytesPerInt);
3933   addi(z, t0, 4);
3934   subw(xstart, xstart, 1); // i = xstart-1;
3935   bltz(xstart, L_last_x);
3936 
3937   shadd(t0, xstart, x, t0, LogBytesPerInt);
3938   ld(product_hi, Address(t0, 0));
3939   ror_imm(product_hi, product_hi, 32); // convert big-endian to little-endian
3940 
3941   Label L_third_loop_prologue;
3942   bind(L_third_loop_prologue);
3943 
3944   sd(ylen, Address(sp, wordSize));
3945   sd(x, Address(sp, 2 * wordSize));
3946   sd(xstart, Address(sp, 3 * wordSize));
3947   multiply_128_x_128_loop(y, z, carry, x, jdx, ylen, product,
3948                           tmp2, x_xstart, tmp3, tmp4, tmp6, product_hi);
3949   ld(z, Address(sp, 0));
3950   ld(ylen, Address(sp, wordSize));
3951   ld(x, Address(sp, 2 * wordSize));
3952   ld(xlen, Address(sp, 3 * wordSize)); // copy old xstart -> xlen
3953   addi(sp, sp, 4 * wordSize);
3954 
3955   addiw(tmp3, xlen, 1);
3956   shadd(t0, tmp3, z, t0, LogBytesPerInt);
3957   sw(carry, Address(t0, 0));
3958 
3959   subw(tmp3, tmp3, 1);
3960   bltz(tmp3, L_done);
3961 
3962   srli(carry, carry, 32);
3963   shadd(t0, tmp3, z, t0, LogBytesPerInt);
3964   sw(carry, Address(t0, 0));
3965   j(L_second_loop_aligned);
3966 
3967   // Next infrequent code is moved outside loops.
3968   bind(L_last_x);
3969   lwu(product_hi, Address(x, 0));
3970   j(L_third_loop_prologue);
3971 
3972   bind(L_done);
3973 }
3974 #endif
3975 
3976 // Count bits of trailing zero chars from lsb to msb until first non-zero element.
3977 // For LL case, one byte for one element, so shift 8 bits once, and for other case,
3978 // shift 16 bits once.
3979 void MacroAssembler::ctzc_bit(Register Rd, Register Rs, bool isLL, Register tmp1, Register tmp2) {
3980   if (UseZbb) {
3981     assert_different_registers(Rd, Rs, tmp1);
3982     int step = isLL ? 8 : 16;
3983     ctz(Rd, Rs);
3984     andi(tmp1, Rd, step - 1);
3985     sub(Rd, Rd, tmp1);
3986     return;
3987   }
3988 
3989   assert_different_registers(Rd, Rs, tmp1, tmp2);
3990   Label Loop;
3991   int step = isLL ? 8 : 16;
3992   mv(Rd, -step);
3993   mv(tmp2, Rs);
3994 
3995   bind(Loop);
3996   addi(Rd, Rd, step);
3997   andi(tmp1, tmp2, ((1 << step) - 1));
3998   srli(tmp2, tmp2, step);
3999   beqz(tmp1, Loop);
4000 }
4001 
4002 // This instruction reads adjacent 4 bytes from the lower half of source register,
4003 // inflate into a register, for example:
4004 // Rs: A7A6A5A4A3A2A1A0
4005 // Rd: 00A300A200A100A0
4006 void MacroAssembler::inflate_lo32(Register Rd, Register Rs, Register tmp1, Register tmp2) {
4007   assert_different_registers(Rd, Rs, tmp1, tmp2);
4008 
4009   mv(tmp1, 0xFF000000); // first byte mask at lower word
4010   andr(Rd, Rs, tmp1);
4011   for (int i = 0; i < 2; i++) {
4012     slli(Rd, Rd, wordSize);
4013     srli(tmp1, tmp1, wordSize);
4014     andr(tmp2, Rs, tmp1);
4015     orr(Rd, Rd, tmp2);
4016   }
4017   slli(Rd, Rd, wordSize);
4018   andi(tmp2, Rs, 0xFF); // last byte mask at lower word
4019   orr(Rd, Rd, tmp2);
4020 }
4021 
4022 // This instruction reads adjacent 4 bytes from the upper half of source register,
4023 // inflate into a register, for example:
4024 // Rs: A7A6A5A4A3A2A1A0
4025 // Rd: 00A700A600A500A4
4026 void MacroAssembler::inflate_hi32(Register Rd, Register Rs, Register tmp1, Register tmp2) {
4027   assert_different_registers(Rd, Rs, tmp1, tmp2);
4028   srli(Rs, Rs, 32);   // only upper 32 bits are needed
4029   inflate_lo32(Rd, Rs, tmp1, tmp2);
4030 }
4031 
4032 // The size of the blocks erased by the zero_blocks stub.  We must
4033 // handle anything smaller than this ourselves in zero_words().
4034 const int MacroAssembler::zero_words_block_size = 8;
4035 
4036 // zero_words() is used by C2 ClearArray patterns.  It is as small as
4037 // possible, handling small word counts locally and delegating
4038 // anything larger to the zero_blocks stub.  It is expanded many times
4039 // in compiled code, so it is important to keep it short.
4040 
4041 // ptr:   Address of a buffer to be zeroed.
4042 // cnt:   Count in HeapWords.
4043 //
4044 // ptr, cnt, and t0 are clobbered.
4045 address MacroAssembler::zero_words(Register ptr, Register cnt) {
4046   assert(is_power_of_2(zero_words_block_size), "adjust this");
4047   assert(ptr == x28 && cnt == x29, "mismatch in register usage");
4048   assert_different_registers(cnt, t0);
4049 
4050   BLOCK_COMMENT("zero_words {");
4051 
4052   mv(t0, zero_words_block_size);
4053   Label around, done, done16;
4054   bltu(cnt, t0, around);
4055   {
4056     RuntimeAddress zero_blocks = RuntimeAddress(StubRoutines::riscv::zero_blocks());
4057     assert(zero_blocks.target() != nullptr, "zero_blocks stub has not been generated");
4058     if (StubRoutines::riscv::complete()) {
4059       address tpc = trampoline_call(zero_blocks);
4060       if (tpc == nullptr) {
4061         DEBUG_ONLY(reset_labels(around));
4062         postcond(pc() == badAddress);
4063         return nullptr;
4064       }
4065     } else {
4066       jal(zero_blocks);
4067     }
4068   }
4069   bind(around);
4070   for (int i = zero_words_block_size >> 1; i > 1; i >>= 1) {
4071     Label l;
4072     test_bit(t0, cnt, exact_log2(i));
4073     beqz(t0, l);
4074     for (int j = 0; j < i; j++) {
4075       sd(zr, Address(ptr, j * wordSize));
4076     }
4077     addi(ptr, ptr, i * wordSize);
4078     bind(l);
4079   }
4080   {
4081     Label l;
4082     test_bit(t0, cnt, 0);
4083     beqz(t0, l);
4084     sd(zr, Address(ptr, 0));
4085     bind(l);
4086   }
4087 
4088   BLOCK_COMMENT("} zero_words");
4089   postcond(pc() != badAddress);
4090   return pc();
4091 }
4092 
4093 #define SmallArraySize (18 * BytesPerLong)
4094 
4095 // base:  Address of a buffer to be zeroed, 8 bytes aligned.
4096 // cnt:   Immediate count in HeapWords.
4097 void MacroAssembler::zero_words(Register base, uint64_t cnt) {
4098   assert_different_registers(base, t0, t1);
4099 
4100   BLOCK_COMMENT("zero_words {");
4101 
4102   if (cnt <= SmallArraySize / BytesPerLong) {
4103     for (int i = 0; i < (int)cnt; i++) {
4104       sd(zr, Address(base, i * wordSize));
4105     }
4106   } else {
4107     const int unroll = 8; // Number of sd(zr, adr), instructions we'll unroll
4108     int remainder = cnt % unroll;
4109     for (int i = 0; i < remainder; i++) {
4110       sd(zr, Address(base, i * wordSize));
4111     }
4112 
4113     Label loop;
4114     Register cnt_reg = t0;
4115     Register loop_base = t1;
4116     cnt = cnt - remainder;
4117     mv(cnt_reg, cnt);
4118     add(loop_base, base, remainder * wordSize);
4119     bind(loop);
4120     sub(cnt_reg, cnt_reg, unroll);
4121     for (int i = 0; i < unroll; i++) {
4122       sd(zr, Address(loop_base, i * wordSize));
4123     }
4124     add(loop_base, loop_base, unroll * wordSize);
4125     bnez(cnt_reg, loop);
4126   }
4127 
4128   BLOCK_COMMENT("} zero_words");
4129 }
4130 
4131 // base:   Address of a buffer to be filled, 8 bytes aligned.
4132 // cnt:    Count in 8-byte unit.
4133 // value:  Value to be filled with.
4134 // base will point to the end of the buffer after filling.
4135 void MacroAssembler::fill_words(Register base, Register cnt, Register value) {
4136 //  Algorithm:
4137 //
4138 //    t0 = cnt & 7
4139 //    cnt -= t0
4140 //    p += t0
4141 //    switch (t0):
4142 //      switch start:
4143 //      do while cnt
4144 //        cnt -= 8
4145 //          p[-8] = value
4146 //        case 7:
4147 //          p[-7] = value
4148 //        case 6:
4149 //          p[-6] = value
4150 //          // ...
4151 //        case 1:
4152 //          p[-1] = value
4153 //        case 0:
4154 //          p += 8
4155 //      do-while end
4156 //    switch end
4157 
4158   assert_different_registers(base, cnt, value, t0, t1);
4159 
4160   Label fini, skip, entry, loop;
4161   const int unroll = 8; // Number of sd instructions we'll unroll
4162 
4163   beqz(cnt, fini);
4164 
4165   andi(t0, cnt, unroll - 1);
4166   sub(cnt, cnt, t0);
4167   // align 8, so first sd n % 8 = mod, next loop sd 8 * n.
4168   shadd(base, t0, base, t1, 3);
4169   la(t1, entry);
4170   slli(t0, t0, 2); // sd_inst_nums * 4; t0 is cnt % 8, so t1 = t1 - sd_inst_nums * 4, 4 is sizeof(inst)
4171   sub(t1, t1, t0);
4172   jr(t1);
4173 
4174   bind(loop);
4175   add(base, base, unroll * 8);
4176   for (int i = -unroll; i < 0; i++) {
4177     sd(value, Address(base, i * 8));
4178   }
4179   bind(entry);
4180   sub(cnt, cnt, unroll);
4181   bgez(cnt, loop);
4182 
4183   bind(fini);
4184 }
4185 
4186 // Zero blocks of memory by using CBO.ZERO.
4187 //
4188 // Aligns the base address first sufficiently for CBO.ZERO, then uses
4189 // CBO.ZERO repeatedly for every full block.  cnt is the size to be
4190 // zeroed in HeapWords.  Returns the count of words left to be zeroed
4191 // in cnt.
4192 //
4193 // NOTE: This is intended to be used in the zero_blocks() stub.  If
4194 // you want to use it elsewhere, note that cnt must be >= CacheLineSize.
4195 void MacroAssembler::zero_dcache_blocks(Register base, Register cnt, Register tmp1, Register tmp2) {
4196   Label initial_table_end, loop;
4197 
4198   // Align base with cache line size.
4199   neg(tmp1, base);
4200   andi(tmp1, tmp1, CacheLineSize - 1);
4201 
4202   // tmp1: the number of bytes to be filled to align the base with cache line size.
4203   add(base, base, tmp1);
4204   srai(tmp2, tmp1, 3);
4205   sub(cnt, cnt, tmp2);
4206   srli(tmp2, tmp1, 1);
4207   la(tmp1, initial_table_end);
4208   sub(tmp2, tmp1, tmp2);
4209   jr(tmp2);
4210   for (int i = -CacheLineSize + wordSize; i < 0; i += wordSize) {
4211     sd(zr, Address(base, i));
4212   }
4213   bind(initial_table_end);
4214 
4215   mv(tmp1, CacheLineSize / wordSize);
4216   bind(loop);
4217   cbo_zero(base);
4218   sub(cnt, cnt, tmp1);
4219   add(base, base, CacheLineSize);
4220   bge(cnt, tmp1, loop);
4221 }
4222 
4223 // java.lang.Math.round(float a)
4224 // Returns the closest int to the argument, with ties rounding to positive infinity.
4225 void MacroAssembler::java_round_float(Register dst, FloatRegister src, FloatRegister ftmp) {
4226   // this instructions calling sequence provides performance improvement on all tested devices;
4227   // don't change it without re-verification
4228   Label done;
4229   mv(t0, jint_cast(0.5f));
4230   fmv_w_x(ftmp, t0);
4231 
4232   // dst = 0 if NaN
4233   feq_s(t0, src, src); // replacing fclass with feq as performance optimization
4234   mv(dst, zr);
4235   beqz(t0, done);
4236 
4237   // dst = (src + 0.5f) rounded down towards negative infinity
4238   //   Adding 0.5f to some floats exceeds the precision limits for a float and rounding takes place.
4239   //   RDN is required for fadd_s, RNE gives incorrect results:
4240   //     --------------------------------------------------------------------
4241   //     fadd.s rne (src + 0.5f): src = 8388609.000000  ftmp = 8388610.000000
4242   //     fcvt.w.s rdn: ftmp = 8388610.000000 dst = 8388610
4243   //     --------------------------------------------------------------------
4244   //     fadd.s rdn (src + 0.5f): src = 8388609.000000  ftmp = 8388609.000000
4245   //     fcvt.w.s rdn: ftmp = 8388609.000000 dst = 8388609
4246   //     --------------------------------------------------------------------
4247   fadd_s(ftmp, src, ftmp, RoundingMode::rdn);
4248   fcvt_w_s(dst, ftmp, RoundingMode::rdn);
4249 
4250   bind(done);
4251 }
4252 
4253 // java.lang.Math.round(double a)
4254 // Returns the closest long to the argument, with ties rounding to positive infinity.
4255 void MacroAssembler::java_round_double(Register dst, FloatRegister src, FloatRegister ftmp) {
4256   // this instructions calling sequence provides performance improvement on all tested devices;
4257   // don't change it without re-verification
4258   Label done;
4259   mv(t0, julong_cast(0.5));
4260   fmv_d_x(ftmp, t0);
4261 
4262   // dst = 0 if NaN
4263   feq_d(t0, src, src); // replacing fclass with feq as performance optimization
4264   mv(dst, zr);
4265   beqz(t0, done);
4266 
4267   // dst = (src + 0.5) rounded down towards negative infinity
4268   fadd_d(ftmp, src, ftmp, RoundingMode::rdn); // RDN is required here otherwise some inputs produce incorrect results
4269   fcvt_l_d(dst, ftmp, RoundingMode::rdn);
4270 
4271   bind(done);
4272 }
4273 
4274 #define FCVT_SAFE(FLOATCVT, FLOATSIG)                                                     \
4275 void MacroAssembler::FLOATCVT##_safe(Register dst, FloatRegister src, Register tmp) {     \
4276   Label done;                                                                             \
4277   assert_different_registers(dst, tmp);                                                   \
4278   fclass_##FLOATSIG(tmp, src);                                                            \
4279   mv(dst, zr);                                                                            \
4280   /* check if src is NaN */                                                               \
4281   andi(tmp, tmp, 0b1100000000);                                                           \
4282   bnez(tmp, done);                                                                        \
4283   FLOATCVT(dst, src);                                                                     \
4284   bind(done);                                                                             \
4285 }
4286 
4287 FCVT_SAFE(fcvt_w_s, s);
4288 FCVT_SAFE(fcvt_l_s, s);
4289 FCVT_SAFE(fcvt_w_d, d);
4290 FCVT_SAFE(fcvt_l_d, d);
4291 
4292 #undef FCVT_SAFE
4293 
4294 #define FCMP(FLOATTYPE, FLOATSIG)                                                       \
4295 void MacroAssembler::FLOATTYPE##_compare(Register result, FloatRegister Rs1,            \
4296                                          FloatRegister Rs2, int unordered_result) {     \
4297   Label Ldone;                                                                          \
4298   if (unordered_result < 0) {                                                           \
4299     /* we want -1 for unordered or less than, 0 for equal and 1 for greater than. */    \
4300     /* installs 1 if gt else 0 */                                                       \
4301     flt_##FLOATSIG(result, Rs2, Rs1);                                                   \
4302     /* Rs1 > Rs2, install 1 */                                                          \
4303     bgtz(result, Ldone);                                                                \
4304     feq_##FLOATSIG(result, Rs1, Rs2);                                                   \
4305     addi(result, result, -1);                                                           \
4306     /* Rs1 = Rs2, install 0 */                                                          \
4307     /* NaN or Rs1 < Rs2, install -1 */                                                  \
4308     bind(Ldone);                                                                        \
4309   } else {                                                                              \
4310     /* we want -1 for less than, 0 for equal and 1 for unordered or greater than. */    \
4311     /* installs 1 if gt or unordered else 0 */                                          \
4312     flt_##FLOATSIG(result, Rs1, Rs2);                                                   \
4313     /* Rs1 < Rs2, install -1 */                                                         \
4314     bgtz(result, Ldone);                                                                \
4315     feq_##FLOATSIG(result, Rs1, Rs2);                                                   \
4316     addi(result, result, -1);                                                           \
4317     /* Rs1 = Rs2, install 0 */                                                          \
4318     /* NaN or Rs1 > Rs2, install 1 */                                                   \
4319     bind(Ldone);                                                                        \
4320     neg(result, result);                                                                \
4321   }                                                                                     \
4322 }
4323 
4324 FCMP(float, s);
4325 FCMP(double, d);
4326 
4327 #undef FCMP
4328 
4329 // Zero words; len is in bytes
4330 // Destroys all registers except addr
4331 // len must be a nonzero multiple of wordSize
4332 void MacroAssembler::zero_memory(Register addr, Register len, Register tmp) {
4333   assert_different_registers(addr, len, tmp, t0, t1);
4334 
4335 #ifdef ASSERT
4336   {
4337     Label L;
4338     andi(t0, len, BytesPerWord - 1);
4339     beqz(t0, L);
4340     stop("len is not a multiple of BytesPerWord");
4341     bind(L);
4342   }
4343 #endif // ASSERT
4344 
4345 #ifndef PRODUCT
4346   block_comment("zero memory");
4347 #endif // PRODUCT
4348 
4349   Label loop;
4350   Label entry;
4351 
4352   // Algorithm:
4353   //
4354   //  t0 = cnt & 7
4355   //  cnt -= t0
4356   //  p += t0
4357   //  switch (t0) {
4358   //    do {
4359   //      cnt -= 8
4360   //        p[-8] = 0
4361   //      case 7:
4362   //        p[-7] = 0
4363   //      case 6:
4364   //        p[-6] = 0
4365   //        ...
4366   //      case 1:
4367   //        p[-1] = 0
4368   //      case 0:
4369   //        p += 8
4370   //     } while (cnt)
4371   //  }
4372 
4373   const int unroll = 8;   // Number of sd(zr) instructions we'll unroll
4374 
4375   srli(len, len, LogBytesPerWord);
4376   andi(t0, len, unroll - 1);  // t0 = cnt % unroll
4377   sub(len, len, t0);          // cnt -= unroll
4378   // tmp always points to the end of the region we're about to zero
4379   shadd(tmp, t0, addr, t1, LogBytesPerWord);
4380   la(t1, entry);
4381   slli(t0, t0, 2);
4382   sub(t1, t1, t0);
4383   jr(t1);
4384   bind(loop);
4385   sub(len, len, unroll);
4386   for (int i = -unroll; i < 0; i++) {
4387     sd(zr, Address(tmp, i * wordSize));
4388   }
4389   bind(entry);
4390   add(tmp, tmp, unroll * wordSize);
4391   bnez(len, loop);
4392 }
4393 
4394 // shift left by shamt and add
4395 // Rd = (Rs1 << shamt) + Rs2
4396 void MacroAssembler::shadd(Register Rd, Register Rs1, Register Rs2, Register tmp, int shamt) {
4397   if (UseZba) {
4398     if (shamt == 1) {
4399       sh1add(Rd, Rs1, Rs2);
4400       return;
4401     } else if (shamt == 2) {
4402       sh2add(Rd, Rs1, Rs2);
4403       return;
4404     } else if (shamt == 3) {
4405       sh3add(Rd, Rs1, Rs2);
4406       return;
4407     }
4408   }
4409 
4410   if (shamt != 0) {
4411     slli(tmp, Rs1, shamt);
4412     add(Rd, Rs2, tmp);
4413   } else {
4414     add(Rd, Rs1, Rs2);
4415   }
4416 }
4417 
4418 void MacroAssembler::zero_extend(Register dst, Register src, int bits) {
4419   if (UseZba && bits == 32) {
4420     zext_w(dst, src);
4421     return;
4422   }
4423 
4424   if (UseZbb && bits == 16) {
4425     zext_h(dst, src);
4426     return;
4427   }
4428 
4429   if (bits == 8) {
4430     zext_b(dst, src);
4431   } else {
4432     slli(dst, src, XLEN - bits);
4433     srli(dst, dst, XLEN - bits);
4434   }
4435 }
4436 
4437 void MacroAssembler::sign_extend(Register dst, Register src, int bits) {
4438   if (UseZbb) {
4439     if (bits == 8) {
4440       sext_b(dst, src);
4441       return;
4442     } else if (bits == 16) {
4443       sext_h(dst, src);
4444       return;
4445     }
4446   }
4447 
4448   if (bits == 32) {
4449     sext_w(dst, src);
4450   } else {
4451     slli(dst, src, XLEN - bits);
4452     srai(dst, dst, XLEN - bits);
4453   }
4454 }
4455 
4456 void MacroAssembler::cmp_l2i(Register dst, Register src1, Register src2, Register tmp)
4457 {
4458   if (src1 == src2) {
4459     mv(dst, zr);
4460     return;
4461   }
4462   Label done;
4463   Register left = src1;
4464   Register right = src2;
4465   if (dst == src1) {
4466     assert_different_registers(dst, src2, tmp);
4467     mv(tmp, src1);
4468     left = tmp;
4469   } else if (dst == src2) {
4470     assert_different_registers(dst, src1, tmp);
4471     mv(tmp, src2);
4472     right = tmp;
4473   }
4474 
4475   // installs 1 if gt else 0
4476   slt(dst, right, left);
4477   bnez(dst, done);
4478   slt(dst, left, right);
4479   // dst = -1 if lt; else if eq , dst = 0
4480   neg(dst, dst);
4481   bind(done);
4482 }
4483 
4484 // The java_calling_convention describes stack locations as ideal slots on
4485 // a frame with no abi restrictions. Since we must observe abi restrictions
4486 // (like the placement of the register window) the slots must be biased by
4487 // the following value.
4488 static int reg2offset_in(VMReg r) {
4489   // Account for saved fp and ra
4490   // This should really be in_preserve_stack_slots
4491   return r->reg2stack() * VMRegImpl::stack_slot_size;
4492 }
4493 
4494 static int reg2offset_out(VMReg r) {
4495   return (r->reg2stack() + SharedRuntime::out_preserve_stack_slots()) * VMRegImpl::stack_slot_size;
4496 }
4497 
4498 // On 64 bit we will store integer like items to the stack as
4499 // 64 bits items (riscv64 abi) even though java would only store
4500 // 32bits for a parameter. On 32bit it will simply be 32 bits
4501 // So this routine will do 32->32 on 32bit and 32->64 on 64bit
4502 void MacroAssembler::move32_64(VMRegPair src, VMRegPair dst, Register tmp) {
4503   if (src.first()->is_stack()) {
4504     if (dst.first()->is_stack()) {
4505       // stack to stack
4506       ld(tmp, Address(fp, reg2offset_in(src.first())));
4507       sd(tmp, Address(sp, reg2offset_out(dst.first())));
4508     } else {
4509       // stack to reg
4510       lw(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first())));
4511     }
4512   } else if (dst.first()->is_stack()) {
4513     // reg to stack
4514     sd(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first())));
4515   } else {
4516     if (dst.first() != src.first()) {
4517       sign_extend(dst.first()->as_Register(), src.first()->as_Register(), 32);
4518     }
4519   }
4520 }
4521 
4522 // An oop arg. Must pass a handle not the oop itself
4523 void MacroAssembler::object_move(OopMap* map,
4524                                  int oop_handle_offset,
4525                                  int framesize_in_slots,
4526                                  VMRegPair src,
4527                                  VMRegPair dst,
4528                                  bool is_receiver,
4529                                  int* receiver_offset) {
4530   assert_cond(map != nullptr && receiver_offset != nullptr);
4531 
4532   // must pass a handle. First figure out the location we use as a handle
4533   Register rHandle = dst.first()->is_stack() ? t1 : dst.first()->as_Register();
4534 
4535   // See if oop is null if it is we need no handle
4536 
4537   if (src.first()->is_stack()) {
4538     // Oop is already on the stack as an argument
4539     int offset_in_older_frame = src.first()->reg2stack() + SharedRuntime::out_preserve_stack_slots();
4540     map->set_oop(VMRegImpl::stack2reg(offset_in_older_frame + framesize_in_slots));
4541     if (is_receiver) {
4542       *receiver_offset = (offset_in_older_frame + framesize_in_slots) * VMRegImpl::stack_slot_size;
4543     }
4544 
4545     ld(t0, Address(fp, reg2offset_in(src.first())));
4546     la(rHandle, Address(fp, reg2offset_in(src.first())));
4547     // conditionally move a null
4548     Label notZero1;
4549     bnez(t0, notZero1);
4550     mv(rHandle, zr);
4551     bind(notZero1);
4552   } else {
4553 
4554     // Oop is in a register we must store it to the space we reserve
4555     // on the stack for oop_handles and pass a handle if oop is non-null
4556 
4557     const Register rOop = src.first()->as_Register();
4558     int oop_slot = -1;
4559     if (rOop == j_rarg0) {
4560       oop_slot = 0;
4561     } else if (rOop == j_rarg1) {
4562       oop_slot = 1;
4563     } else if (rOop == j_rarg2) {
4564       oop_slot = 2;
4565     } else if (rOop == j_rarg3) {
4566       oop_slot = 3;
4567     } else if (rOop == j_rarg4) {
4568       oop_slot = 4;
4569     } else if (rOop == j_rarg5) {
4570       oop_slot = 5;
4571     } else if (rOop == j_rarg6) {
4572       oop_slot = 6;
4573     } else {
4574       assert(rOop == j_rarg7, "wrong register");
4575       oop_slot = 7;
4576     }
4577 
4578     oop_slot = oop_slot * VMRegImpl::slots_per_word + oop_handle_offset;
4579     int offset = oop_slot * VMRegImpl::stack_slot_size;
4580 
4581     map->set_oop(VMRegImpl::stack2reg(oop_slot));
4582     // Store oop in handle area, may be null
4583     sd(rOop, Address(sp, offset));
4584     if (is_receiver) {
4585       *receiver_offset = offset;
4586     }
4587 
4588     //rOop maybe the same as rHandle
4589     if (rOop == rHandle) {
4590       Label isZero;
4591       beqz(rOop, isZero);
4592       la(rHandle, Address(sp, offset));
4593       bind(isZero);
4594     } else {
4595       Label notZero2;
4596       la(rHandle, Address(sp, offset));
4597       bnez(rOop, notZero2);
4598       mv(rHandle, zr);
4599       bind(notZero2);
4600     }
4601   }
4602 
4603   // If arg is on the stack then place it otherwise it is already in correct reg.
4604   if (dst.first()->is_stack()) {
4605     sd(rHandle, Address(sp, reg2offset_out(dst.first())));
4606   }
4607 }
4608 
4609 // A float arg may have to do float reg int reg conversion
4610 void MacroAssembler::float_move(VMRegPair src, VMRegPair dst, Register tmp) {
4611   assert(src.first()->is_stack() && dst.first()->is_stack() ||
4612          src.first()->is_reg() && dst.first()->is_reg() ||
4613          src.first()->is_stack() && dst.first()->is_reg(), "Unexpected error");
4614   if (src.first()->is_stack()) {
4615     if (dst.first()->is_stack()) {
4616       lwu(tmp, Address(fp, reg2offset_in(src.first())));
4617       sw(tmp, Address(sp, reg2offset_out(dst.first())));
4618     } else if (dst.first()->is_Register()) {
4619       lwu(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first())));
4620     } else {
4621       ShouldNotReachHere();
4622     }
4623   } else if (src.first() != dst.first()) {
4624     if (src.is_single_phys_reg() && dst.is_single_phys_reg()) {
4625       fmv_s(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
4626     } else {
4627       ShouldNotReachHere();
4628     }
4629   }
4630 }
4631 
4632 // A long move
4633 void MacroAssembler::long_move(VMRegPair src, VMRegPair dst, Register tmp) {
4634   if (src.first()->is_stack()) {
4635     if (dst.first()->is_stack()) {
4636       // stack to stack
4637       ld(tmp, Address(fp, reg2offset_in(src.first())));
4638       sd(tmp, Address(sp, reg2offset_out(dst.first())));
4639     } else {
4640       // stack to reg
4641       ld(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first())));
4642     }
4643   } else if (dst.first()->is_stack()) {
4644     // reg to stack
4645     sd(src.first()->as_Register(), Address(sp, reg2offset_out(dst.first())));
4646   } else {
4647     if (dst.first() != src.first()) {
4648       mv(dst.first()->as_Register(), src.first()->as_Register());
4649     }
4650   }
4651 }
4652 
4653 // A double move
4654 void MacroAssembler::double_move(VMRegPair src, VMRegPair dst, Register tmp) {
4655   assert(src.first()->is_stack() && dst.first()->is_stack() ||
4656          src.first()->is_reg() && dst.first()->is_reg() ||
4657          src.first()->is_stack() && dst.first()->is_reg(), "Unexpected error");
4658   if (src.first()->is_stack()) {
4659     if (dst.first()->is_stack()) {
4660       ld(tmp, Address(fp, reg2offset_in(src.first())));
4661       sd(tmp, Address(sp, reg2offset_out(dst.first())));
4662     } else if (dst.first()-> is_Register()) {
4663       ld(dst.first()->as_Register(), Address(fp, reg2offset_in(src.first())));
4664     } else {
4665       ShouldNotReachHere();
4666     }
4667   } else if (src.first() != dst.first()) {
4668     if (src.is_single_phys_reg() && dst.is_single_phys_reg()) {
4669       fmv_d(dst.first()->as_FloatRegister(), src.first()->as_FloatRegister());
4670     } else {
4671       ShouldNotReachHere();
4672     }
4673   }
4674 }
4675 
4676 void MacroAssembler::rt_call(address dest, Register tmp) {
4677   CodeBlob *cb = CodeCache::find_blob(dest);
4678   RuntimeAddress target(dest);
4679   if (cb) {
4680     far_call(target);
4681   } else {
4682     relocate(target.rspec(), [&] {
4683       int32_t offset;
4684       la_patchable(tmp, target, offset);
4685       jalr(x1, tmp, offset);
4686     });
4687   }
4688 }
4689 
4690 void MacroAssembler::test_bit(Register Rd, Register Rs, uint32_t bit_pos) {
4691   assert(bit_pos < 64, "invalid bit range");
4692   if (UseZbs) {
4693     bexti(Rd, Rs, bit_pos);
4694     return;
4695   }
4696   int64_t imm = (int64_t)(1UL << bit_pos);
4697   if (is_simm12(imm)) {
4698     and_imm12(Rd, Rs, imm);
4699   } else {
4700     srli(Rd, Rs, bit_pos);
4701     and_imm12(Rd, Rd, 1);
4702   }
4703 }
4704 
4705 // Implements lightweight-locking.
4706 // Branches to slow upon failure to lock the object.
4707 // Falls through upon success.
4708 //
4709 //  - obj: the object to be locked
4710 //  - hdr: the header, already loaded from obj, will be destroyed
4711 //  - tmp1, tmp2: temporary registers, will be destroyed
4712 void MacroAssembler::lightweight_lock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow) {
4713   assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
4714   assert_different_registers(obj, hdr, tmp1, tmp2, t0);
4715 
4716   // Check if we would have space on lock-stack for the object.
4717   lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
4718   mv(tmp2, (unsigned)LockStack::end_offset());
4719   bge(tmp1, tmp2, slow, /* is_far */ true);
4720 
4721   // Load (object->mark() | 1) into hdr
4722   ori(hdr, hdr, markWord::unlocked_value);
4723   // Clear lock-bits, into tmp2
4724   xori(tmp2, hdr, markWord::unlocked_value);
4725 
4726   // Try to swing header from unlocked to locked
4727   Label success;
4728   cmpxchgptr(hdr, tmp2, obj, tmp1, success, &slow);
4729   bind(success);
4730 
4731   // After successful lock, push object on lock-stack
4732   lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
4733   add(tmp2, xthread, tmp1);
4734   sd(obj, Address(tmp2, 0));
4735   addw(tmp1, tmp1, oopSize);
4736   sw(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
4737 }
4738 
4739 // Implements ligthweight-unlocking.
4740 // Branches to slow upon failure.
4741 // Falls through upon success.
4742 //
4743 // - obj: the object to be unlocked
4744 // - hdr: the (pre-loaded) header of the object
4745 // - tmp1, tmp2: temporary registers
4746 void MacroAssembler::lightweight_unlock(Register obj, Register hdr, Register tmp1, Register tmp2, Label& slow) {
4747   assert(LockingMode == LM_LIGHTWEIGHT, "only used with new lightweight locking");
4748   assert_different_registers(obj, hdr, tmp1, tmp2, t0);
4749 
4750 #ifdef ASSERT
4751   {
4752     // The following checks rely on the fact that LockStack is only ever modified by
4753     // its owning thread, even if the lock got inflated concurrently; removal of LockStack
4754     // entries after inflation will happen delayed in that case.
4755 
4756     // Check for lock-stack underflow.
4757     Label stack_ok;
4758     lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
4759     mv(tmp2, (unsigned)LockStack::start_offset());
4760     bgt(tmp1, tmp2, stack_ok);
4761     STOP("Lock-stack underflow");
4762     bind(stack_ok);
4763   }
4764   {
4765     // Check if the top of the lock-stack matches the unlocked object.
4766     Label tos_ok;
4767     subw(tmp1, tmp1, oopSize);
4768     add(tmp1, xthread, tmp1);
4769     ld(tmp1, Address(tmp1, 0));
4770     beq(tmp1, obj, tos_ok);
4771     STOP("Top of lock-stack does not match the unlocked object");
4772     bind(tos_ok);
4773   }
4774   {
4775     // Check that hdr is fast-locked.
4776    Label hdr_ok;
4777     andi(tmp1, hdr, markWord::lock_mask_in_place);
4778     beqz(tmp1, hdr_ok);
4779     STOP("Header is not fast-locked");
4780     bind(hdr_ok);
4781   }
4782 #endif
4783 
4784   // Load the new header (unlocked) into tmp1
4785   ori(tmp1, hdr, markWord::unlocked_value);
4786 
4787   // Try to swing header from locked to unlocked
4788   Label success;
4789   cmpxchgptr(hdr, tmp1, obj, tmp2, success, &slow);
4790   bind(success);
4791 
4792   // After successful unlock, pop object from lock-stack
4793   lwu(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
4794   subw(tmp1, tmp1, oopSize);
4795 #ifdef ASSERT
4796   add(tmp2, xthread, tmp1);
4797   sd(zr, Address(tmp2, 0));
4798 #endif
4799   sw(tmp1, Address(xthread, JavaThread::lock_stack_top_offset()));
4800 }